From b437c34f924a689177b3ab8b867f78ed6b5edb95 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:22:20 -0400 Subject: [PATCH 01/27] fix(controlplane): recover a lost dispatch instead of wedging the deploy pipeline A Deployment row left in Dispatching with no NodeDeploymentState rows (HandleDispatch threw before its SaveChanges, or the process died between the AdminOps row write and the coordinator's Tell) was recovered with an empty expected set; the deadline then hit the 0 == 0 "race" branch and returned without touching the row, so every later StartDeployment was refused AnotherDeploymentInFlight until someone hand-edited SQL. - PreStart: a recovered row with no expected-ack rows is a lost dispatch. Inside the apply deadline it is re-driven through HandleDispatch (bounded: CreatedAtUtc never moves); past it the row is closed TimedOut with a reason naming the condition. - HandleDispatch is idempotent on a re-dispatch (rows already seeded ARE the expected set). - HandleDeadline takes the race branch only when _expectedAcks.Count > 0. - The PreStart recovery read projects ArtifactBlob away (R03:3.5). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Coordinators/ConfigPublishCoordinator.cs | 99 ++++++-- .../ConfigPublishCoordinatorRecoveryTests.cs | 234 ++++++++++++++++++ 2 files changed, 316 insertions(+), 17 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index b621cc64..3c99e998 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -161,9 +161,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(DeploymentAcksTopic, Self)); using var db = _dbFactory.CreateDbContext(); + // Projected: the row carries the whole ArtifactBlob (varbinary(max)), which this recovery never + // reads — and this query runs on every singleton start and hand-over. var inflight = db.Deployments .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, d.RevisionHash, d.CreatedAtUtc }) .FirstOrDefault(); if (inflight is null) return; @@ -173,6 +176,41 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers .AsNoTracking() .ToList(); + // Resume the deadline timer using the remaining time. The deadline runs from when the + // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough + // proxy — we don't track a separate "dispatched at" column). + var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc; + var remaining = _applyDeadline - elapsed; + + if (nodeStates.Count == 0) + { + // The expected set and the AwaitingApplyAcks flip are written in ONE SaveChanges by + // HandleDispatch, so a row with no NodeDeploymentState rows is a dispatch that never completed: + // HandleDispatch threw before its SaveChanges (this actor restarted and the message was + // dropped), the process died between AdminOps' row write and the coordinator's Tell, or the + // Tell was lost across a singleton hand-over. Recovering it with an EMPTY expected set used to + // let the deadline take the "everyone acked" race branch (0 == 0) and leave the row in flight + // forever, refusing every later StartDeployment. Inside the deadline the dispatch is simply + // re-driven through the normal handler (bounded: CreatedAtUtc never moves, so a dispatch that + // keeps throwing ends up on the other branch once the deadline passes); past it the row is + // closed as TimedOut with a reason that names the condition. + if (remaining > TimeSpan.Zero) + { + _log.Warning( + "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " + + "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " + + "re-dispatching with {Remaining} of the apply deadline left", + _current, remaining); + Self.Tell(new DispatchDeployment( + _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId())); + } + else + { + Self.Tell(new DeadlineElapsed(_current.Value)); + } + return; + } + _expectedAcks = nodeStates.Select(s => NodeId.Parse(s.NodeId)).ToHashSet(NodeIdComparer); foreach (var s in nodeStates.Where(s => s.Status != NodeDeploymentStatus.Applying)) { @@ -182,11 +220,6 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId)); } - // Resume the deadline timer using the remaining time. The deadline runs from when the - // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough - // proxy — we don't track a separate "dispatched at" column). - var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc; - var remaining = _applyDeadline - elapsed; if (remaining <= TimeSpan.Zero) { Self.Tell(new DeadlineElapsed(_current.Value)); @@ -208,19 +241,34 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // Seed NodeDeploymentState rows so a failover knows which nodes were expected to ack. using (var db = _dbFactory.CreateDbContext()) { - // Discovery reuses this context rather than opening a second one — it reads the same - // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single - // context keeps the expected set and the seeded rows consistent by construction. - _expectedAcks = DiscoverDriverNodes(db); - - foreach (var node in _expectedAcks) + // Idempotent on a re-dispatch (PreStart re-drives a lost dispatch through this handler): rows + // already seeded for this deployment ARE the expected set — re-discovering would both duplicate + // the primary key and let a node toggled since the first attempt change who is expected. + var seeded = db.NodeDeploymentStates + .AsNoTracking() + .Where(x => x.DeploymentId == msg.DeploymentId.Value) + .Select(x => x.NodeId) + .ToList(); + if (seeded.Count > 0) { - db.NodeDeploymentStates.Add(new NodeDeploymentState + _expectedAcks = seeded.Select(NodeId.Parse).ToHashSet(NodeIdComparer); + } + else + { + // Discovery reuses this context rather than opening a second one — it reads the same + // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single + // context keeps the expected set and the seeded rows consistent by construction. + _expectedAcks = DiscoverDriverNodes(db); + + foreach (var node in _expectedAcks) { - NodeId = node.Value, - DeploymentId = msg.DeploymentId.Value, - Status = NodeDeploymentStatus.Applying, - }); + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = node.Value, + DeploymentId = msg.DeploymentId.Value, + Status = NodeDeploymentStatus.Applying, + }); + } } UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks); db.SaveChanges(); @@ -360,7 +408,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers msg.DeploymentId, _current); return; } - if (_acks.Count == _expectedAcks.Count) + // Only a NON-EMPTY expected set can be "fully acked": with nothing expected, 0 == 0 is not the + // everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped. + if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count) { // Race: every node acked just as the deadline fired. Already sealed/failed elsewhere. return; @@ -368,6 +418,21 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers using var db = _dbFactory.CreateDbContext(); UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut); + if (_expectedAcks.Count == 0) + { + var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); + if (d is not null) + d.FailureReason = Truncate( + "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + + "hand-over mid-dispatch) and the apply deadline has passed. Redeploy."); + db.SaveChanges(); + _log.Warning( + "Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " + + "expected and the {Deadline} apply deadline has passed. Closed as TimedOut — redeploy", + _current.Value, _applyDeadline); + ResetForNext(); + return; + } db.SaveChanges(); // Name the silent nodes, not just the counts. Since the expected set became DB-derived this // is the failure an operator meets after deploying while a configured node is switched off, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs new file mode 100644 index 00000000..86f6efa7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -0,0 +1,234 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Review 03 §2.2: a Deployment row left in Dispatching with NO NodeDeploymentState +/// rows — HandleDispatch threw before its SaveChanges, or the process died between the +/// AdminOps row write and the coordinator's Tell — used to be recovered with an EMPTY expected +/// set, at which point the deadline hit the 0 == 0 "race" branch and returned without touching +/// the row. Every later StartDeployment was then refused AnotherDeploymentInFlight forever. +/// +public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTestBase +{ + private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64)); + private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5); + + /// + /// A lost dispatch that is still inside the apply deadline is re-driven: PreStart re-dispatches it, + /// the expected set is seeded from the enabled rows, and the node's ack seals it. + /// + [Fact] + public void Recovered_row_with_no_expected_set_is_redispatched_and_then_seals() + { + var dbFactory = NewInMemoryDbFactory(); + var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + SeedNodes(dbFactory, "site-a-1:4053"); + + // No DispatchDeployment is ever sent — PreStart alone has to recover it. + var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" }); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// + /// A lost dispatch that is already past the apply deadline is closed as TimedOut with a reason + /// naming the condition — never left in Dispatching — and the next deployment proceeds + /// (positive control: it is dispatched, acked and sealed through the same coordinator). + /// + [Fact] + public void Recovered_row_past_deadline_with_no_expected_set_is_timed_out_not_wedged() + { + var dbFactory = NewInMemoryDbFactory(); + var staleId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); + SeedNodes(dbFactory, "site-a-1:4053"); + + var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + var row = db.Deployments.Single(d => d.DeploymentId == staleId.Value); + row.Status.ShouldBe(DeploymentStatus.TimedOut); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("lost"); + }, duration: PresenceBudget); + + // Positive control: the coordinator is not wedged — a fresh deployment runs end to end. + var nextId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + actor.Tell(new DispatchDeployment(nextId, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + actor.Tell(new ApplyAck(nextId, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// + /// An AwaitingApplyAcks row with no rows (the process died between the status flip and the + /// empty-fleet seal) is re-driven too, and with no enabled nodes it seals empty exactly as a fresh + /// dispatch would — the recovery keys on the missing expected set, not on the status value. + /// + [Fact] + public void Recovered_AwaitingApplyAcks_row_with_no_rows_and_no_fleet_seals_empty() + { + var dbFactory = NewInMemoryDbFactory(); + SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow); + Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + db.NodeDeploymentStates.ShouldBeEmpty(); + }, duration: PresenceBudget); + } + + /// + /// The brief's scenario end to end: HandleDispatch throws once (a transient fault seeding the + /// expected set). The coordinator restarts, recovers the Dispatching row, re-dispatches it and + /// the deployment completes; a second StartDeployment is then Accepted (positive control: a + /// normal deploy seals). + /// + [Fact] + public async Task HandleDispatch_throw_is_recovered_and_the_next_StartDeployment_succeeds() + { + var dbName = Guid.NewGuid().ToString("N"); + var adminFactory = NewInMemoryDbFactory(dbName); + var faulting = new OneShotFaultingDbFactory(NewInMemoryDbFactory(dbName)); + SeedNodes(adminFactory, "site-a-1:4053"); + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout)); + var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty())); + + // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and + // the actor would be stopped, not restarted). Identify is processed after creation, so a + // resolved selection proves PreStart completed. + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); + faulting.FailNext(); + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var first = ExpectMsg(PresenceBudget); + first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + first.DeploymentId.ShouldNotBeNull(); + + // The fault fired, the actor restarted, PreStart re-dispatched: the expected set lands. + AwaitAssert(() => + { + faulting.Faulted.ShouldBeTrue(); + using var db = adminFactory.CreateDbContext(); + db.NodeDeploymentStates.Count(s => s.DeploymentId == first.DeploymentId.Value.Value).ShouldBe(1); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + coordinator.Tell(new ApplyAck(first.DeploymentId.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = adminFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + + // Positive control: nothing is wedged — the next deploy is Accepted and seals too. + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var second = ExpectMsg(PresenceBudget); + second.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.Tell(new ApplyAck(second.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = adminFactory.CreateDbContext(); + db.Deployments.Count(d => d.Status == DeploymentStatus.Sealed).ShouldBe(2); + }, duration: PresenceBudget); + } + + private static DeploymentId SeedDeployment( + IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) + { + var id = DeploymentId.NewId(); + using var db = dbFactory.CreateDbContext(); + db.Deployments.Add(new Deployment + { + DeploymentId = id.Value, + RevisionHash = TestRevision.Value, + Status = status, + CreatedBy = "test", + CreatedAtUtc = createdAtUtc, + }); + db.SaveChanges(); + return id; + } + + private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodeIds) + { + using var db = dbFactory.CreateDbContext(); + db.ServerClusters.Add(new ServerCluster + { + ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", + NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + foreach (var nodeId in nodeIds) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0], + ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test", + }); + } + db.SaveChanges(); + } + + /// Throws from exactly one CreateDbContext call after . + private sealed class OneShotFaultingDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private int _armed; + public bool Faulted { get; private set; } + + public void FailNext() => Interlocked.Exchange(ref _armed, 1); + + public OtOpcUaConfigDbContext CreateDbContext() + { + if (Interlocked.Exchange(ref _armed, 0) == 1) + { + Faulted = true; + throw new InvalidOperationException("injected transient fault seeding the expected set"); + } + return inner.CreateDbContext(); + } + } +} From 73e74e270ddf8b9be39c010935cc1aff9c8f91fe Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:36:12 -0400 Subject: [PATCH 02/27] fix(controlplane): validate and compose a deployment from ONE snapshot read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StartDeployment read the whole config DB twice with no transaction — DraftSnapshotFactory for the validator, then ConfigComposer for the artifact, plus clusters/nodes and Script.SourceCode again in between. An AdminUI save landing between the two shipped an artifact the DraftValidator never saw. - DraftSnapshot carries ServerClusters, ClusterNodes and ApiKeys; DraftSnapshotFactory reads every table inside one SNAPSHOT transaction on a relational provider (no-op on InMemory or under an ambient transaction). - ConfigComposer.Compose(DraftSnapshot) is pure over the snapshot (ordinal-sorted keys); SnapshotAndFlattenAsync(db) delegates to it, byte-identical. - AdminOperationsActor validates topology, estimates compile cost and composes the artifact from the one snapshot it holds. - Migration EnableSnapshotIsolation: ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON (suppressTransaction; no model change). Tests: the SQL Server snapshot read ignores a row committed mid-read (positive control: a later read sees it); an edit injected between the Equipment and Tags reads is absent from the shipped artifact and rejected by the next deploy; Compose(snapshot) == SnapshotAndFlattenAsync(db). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- ...102459_EnableSnapshotIsolation.Designer.cs | 1618 +++++++++++++++++ .../20261022102459_EnableSnapshotIsolation.cs | 36 + .../Validation/DraftSnapshot.cs | 17 + .../Validation/DraftSnapshotFactory.cs | 31 + .../AdminOperations/AdminOperationsActor.cs | 29 +- .../AdminOperations/ConfigComposer.cs | 61 +- .../DraftSnapshotFactorySqlServerTests.cs | 117 ++ .../DraftSnapshotFactoryTests.cs | 30 + ...AdminOperationsActorSingleSnapshotTests.cs | 133 ++ .../ConfigComposerTests.cs | 41 + 10 files changed, 2077 insertions(+), 36 deletions(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs new file mode 100644 index 00000000..db54f3a9 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs @@ -0,0 +1,1618 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ZB.MOM.WW.OtOpcUa.Configuration; + +#nullable disable + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations +{ + [DbContext(typeof(OtOpcUaConfigDbContext))] + [Migration("20261022102459_EnableSnapshotIsolation")] + partial class EnableSnapshotIsolation + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("Microsoft.AspNetCore.DataProtection.EntityFrameworkCore.DataProtectionKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("FriendlyName") + .HasColumnType("nvarchar(max)"); + + b.Property("Xml") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("DataProtectionKeys", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ApiKey", b => + { + b.Property("ApiKeyRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ApiKeyId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedUtc") + .HasColumnType("datetime2"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("ExpiresUtc") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SecretHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.HasKey("ApiKeyRowId"); + + b.HasIndex("ApiKeyId") + .IsUnique() + .HasDatabaseName("UX_ApiKey_ApiKeyId"); + + b.HasIndex("Principal") + .HasDatabaseName("IX_ApiKey_Principal"); + + b.ToTable("ApiKey", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("AkkaPort") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValue(4053); + + b.Property("ApplicationUri") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("DashboardPort") + .HasColumnType("int"); + + b.Property("DriverConfigOverridesJson") + .HasColumnType("nvarchar(max)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("GrpcPort") + .HasColumnType("int"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("LastSeenAt") + .HasColumnType("datetime2(3)"); + + b.Property("MaintenanceMode") + .HasColumnType("bit"); + + b.Property("OpcUaPort") + .HasColumnType("int"); + + b.Property("ServiceLevelBase") + .HasColumnType("tinyint"); + + b.HasKey("NodeId"); + + b.HasIndex("ApplicationUri") + .IsUnique() + .HasDatabaseName("UX_ClusterNode_ApplicationUri"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_ClusterNode_ClusterId"); + + b.ToTable("ClusterNode", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNodeCredential", b => + { + b.Property("CredentialId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("NodeId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RotatedAt") + .HasColumnType("datetime2(3)"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.HasKey("CredentialId"); + + b.HasIndex("Kind", "Value") + .IsUnique() + .HasDatabaseName("UX_ClusterNodeCredential_Value") + .HasFilter("[Enabled] = 1"); + + b.HasIndex("NodeId", "Enabled") + .HasDatabaseName("IX_ClusterNodeCredential_NodeId"); + + b.ToTable("ClusterNodeCredential", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ConfigAuditLog", b => + { + b.Property("AuditId") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("AuditId")); + + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CorrelationId") + .HasColumnType("uniqueidentifier"); + + b.Property("DetailsJson") + .HasColumnType("nvarchar(max)"); + + b.Property("EventId") + .HasColumnType("uniqueidentifier"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("GenerationId") + .HasColumnType("bigint"); + + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Outcome") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Timestamp") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.HasKey("AuditId"); + + b.HasIndex("EventId") + .IsUnique() + .HasDatabaseName("UX_ConfigAuditLog_EventId") + .HasFilter("[EventId] IS NOT NULL"); + + b.HasIndex("GenerationId") + .HasDatabaseName("IX_ConfigAuditLog_Generation") + .HasFilter("[GenerationId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Timestamp") + .IsDescending(false, true) + .HasDatabaseName("IX_ConfigAuditLog_Cluster_Time"); + + b.ToTable("ConfigAuditLog", null, t => + { + t.HasCheckConstraint("CK_ConfigAuditLog_DetailsJson_IsJson", "DetailsJson IS NULL OR ISJSON(DetailsJson) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ConfigEdit", b => + { + b.Property("EditId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("EditedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("EditedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EntityId") + .HasColumnType("uniqueidentifier"); + + b.Property("EntityType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ExecutionId") + .HasColumnType("uniqueidentifier"); + + b.Property("FieldsJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SourceNode") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("EditId"); + + b.HasIndex("EditedAtUtc") + .HasDatabaseName("IX_ConfigEdit_EditedAt"); + + b.HasIndex("ExecutionId") + .HasDatabaseName("IX_ConfigEdit_Execution") + .HasFilter("[ExecutionId] IS NOT NULL"); + + b.HasIndex("EntityType", "EntityId") + .HasDatabaseName("IX_ConfigEdit_Entity"); + + b.ToTable("ConfigEdit", null, t => + { + t.HasCheckConstraint("CK_ConfigEdit_FieldsJson_IsJson", "ISJSON(FieldsJson) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Deployment", b => + { + b.Property("DeploymentId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ArtifactBlob") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("CreatedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("FailureReason") + .HasMaxLength(2048) + .HasColumnType("nvarchar(2048)"); + + b.Property("RevisionHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SealedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("DeploymentId"); + + b.HasIndex("CreatedAtUtc") + .HasDatabaseName("IX_Deployment_CreatedAt"); + + b.HasIndex("Status") + .HasDatabaseName("IX_Deployment_Status"); + + b.ToTable("Deployment", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Device", b => + { + b.Property("DeviceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DeviceConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DeviceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverInstanceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("DeviceRowId"); + + b.HasIndex("DeviceId") + .IsUnique() + .HasDatabaseName("UX_Device_LogicalId") + .HasFilter("[DeviceId] IS NOT NULL"); + + b.HasIndex("DriverInstanceId") + .HasDatabaseName("IX_Device_Driver"); + + b.HasIndex("DriverInstanceId", "Name") + .IsUnique() + .HasDatabaseName("UX_Device_Driver_Name"); + + b.ToTable("Device", null, t => + { + t.HasCheckConstraint("CK_Device_DeviceConfig_IsJson", "ISJSON(DeviceConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverInstance", b => + { + b.Property("DriverInstanceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DriverInstanceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ResilienceConfig") + .HasColumnType("nvarchar(max)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("DriverInstanceRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_DriverInstance_Cluster"); + + b.HasIndex("DriverInstanceId") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_LogicalId") + .HasFilter("[DriverInstanceId] IS NOT NULL"); + + b.HasIndex("RawFolderId") + .HasDatabaseName("IX_DriverInstance_RawFolder"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_ClusterRoot_Name") + .HasFilter("[RawFolderId] IS NULL"); + + b.HasIndex("ClusterId", "RawFolderId", "Name") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_Folder_Name") + .HasFilter("[RawFolderId] IS NOT NULL"); + + b.ToTable("DriverInstance", null, t => + { + t.HasCheckConstraint("CK_DriverInstance_DriverConfig_IsJson", "ISJSON(DriverConfig) = 1"); + + t.HasCheckConstraint("CK_DriverInstance_ResilienceConfig_IsJson", "ResilienceConfig IS NULL OR ISJSON(ResilienceConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Equipment", b => + { + b.Property("EquipmentRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AssetLocation") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("DeviceManualUri") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentClassRef") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EquipmentId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("EquipmentUuid") + .HasColumnType("uniqueidentifier"); + + b.Property("HardwareRevision") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("MachineCode") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Manufacturer") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ManufacturerUri") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Model") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SAPID") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SerialNumber") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SoftwareRevision") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("UnsLineId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("YearOfConstruction") + .HasColumnType("smallint"); + + b.Property("ZTag") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("EquipmentRowId"); + + b.HasIndex("EquipmentId") + .IsUnique() + .HasDatabaseName("UX_Equipment_LogicalId") + .HasFilter("[EquipmentId] IS NOT NULL"); + + b.HasIndex("EquipmentUuid") + .IsUnique() + .HasDatabaseName("UX_Equipment_Uuid"); + + b.HasIndex("MachineCode") + .HasDatabaseName("IX_Equipment_MachineCode"); + + b.HasIndex("SAPID") + .HasDatabaseName("IX_Equipment_SAPID") + .HasFilter("[SAPID] IS NOT NULL"); + + b.HasIndex("UnsLineId") + .HasDatabaseName("IX_Equipment_Line"); + + b.HasIndex("ZTag") + .HasDatabaseName("IX_Equipment_ZTag") + .HasFilter("[ZTag] IS NOT NULL"); + + b.HasIndex("UnsLineId", "Name") + .IsUnique() + .HasDatabaseName("UX_Equipment_LinePath"); + + b.ToTable("Equipment", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ExternalIdReservation", b => + { + b.Property("ReservationId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("EquipmentUuid") + .HasColumnType("uniqueidentifier"); + + b.Property("FirstPublishedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("FirstPublishedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("LastPublishedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("ReleaseReason") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("ReleasedAt") + .HasColumnType("datetime2(3)"); + + b.Property("ReleasedBy") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("ReservationId"); + + b.HasIndex("EquipmentUuid") + .HasDatabaseName("IX_ExternalIdReservation_Equipment"); + + b.HasIndex("Kind", "Value") + .IsUnique() + .HasDatabaseName("UX_ExternalIdReservation_KindValue_Active") + .HasFilter("[ReleasedAt] IS NULL"); + + b.ToTable("ExternalIdReservation", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.LdapGroupRoleMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("IsSystemWide") + .HasColumnType("bit"); + + b.Property("LdapGroup") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Role") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("Id"); + + b.HasIndex("ClusterId"); + + b.HasIndex("LdapGroup") + .HasDatabaseName("IX_LdapGroupRoleMapping_Group"); + + b.HasIndex("LdapGroup", "ClusterId") + .IsUnique() + .HasDatabaseName("UX_LdapGroupRoleMapping_Group_Cluster") + .HasFilter("[ClusterId] IS NOT NULL"); + + b.ToTable("LdapGroupRoleMapping", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeAcl", b => + { + b.Property("NodeAclRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("NodeAclId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("PermissionFlags") + .HasColumnType("int"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScopeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ScopeKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.HasKey("NodeAclRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_NodeAcl_Cluster"); + + b.HasIndex("NodeAclId") + .IsUnique() + .HasDatabaseName("UX_NodeAcl_LogicalId") + .HasFilter("[NodeAclId] IS NOT NULL"); + + b.HasIndex("Principal") + .HasDatabaseName("IX_NodeAcl_Principal"); + + b.HasIndex("ScopeKind", "ScopeId") + .HasDatabaseName("IX_NodeAcl_Scope") + .HasFilter("[ScopeId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Principal", "ScopeKind", "ScopeId") + .IsUnique() + .HasDatabaseName("UX_NodeAcl_PrincipalScope") + .HasFilter("[ScopeId] IS NOT NULL"); + + b.ToTable("NodeAcl", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeDeploymentState", b => + { + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DeploymentId") + .HasColumnType("uniqueidentifier"); + + b.Property("AppliedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("FailureReason") + .HasMaxLength(2048) + .HasColumnType("nvarchar(2048)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("StartedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("NodeId", "DeploymentId"); + + b.HasIndex("DeploymentId") + .HasDatabaseName("IX_NodeDeploymentState_Deployment"); + + b.HasIndex("Status") + .HasDatabaseName("IX_NodeDeploymentState_Status"); + + b.ToTable("NodeDeploymentState", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.PollGroup", b => + { + b.Property("PollGroupRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DriverInstanceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("IntervalMs") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PollGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("PollGroupRowId"); + + b.HasIndex("DriverInstanceId") + .HasDatabaseName("IX_PollGroup_Driver"); + + b.HasIndex("PollGroupId") + .IsUnique() + .HasDatabaseName("UX_PollGroup_LogicalId") + .HasFilter("[PollGroupId] IS NOT NULL"); + + b.ToTable("PollGroup", null, t => + { + t.HasCheckConstraint("CK_PollGroup_IntervalMs_Min", "IntervalMs >= 50"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.RawFolder", b => + { + b.Property("RawFolderRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ParentRawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.HasKey("RawFolderRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_RawFolder_Cluster"); + + b.HasIndex("RawFolderId") + .IsUnique() + .HasDatabaseName("UX_RawFolder_LogicalId") + .HasFilter("[RawFolderId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_RawFolder_ClusterRoot_Name") + .HasFilter("[ParentRawFolderId] IS NULL"); + + b.HasIndex("ClusterId", "ParentRawFolderId", "Name") + .IsUnique() + .HasDatabaseName("UX_RawFolder_Parent_Name") + .HasFilter("[ParentRawFolderId] IS NOT NULL"); + + b.ToTable("RawFolder", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Script", b => + { + b.Property("ScriptRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("Language") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SourceCode") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SourceHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("ScriptRowId"); + + b.HasIndex("ScriptId") + .IsUnique() + .HasDatabaseName("UX_Script_LogicalId") + .HasFilter("[ScriptId] IS NOT NULL"); + + b.HasIndex("SourceHash") + .HasDatabaseName("IX_Script_SourceHash"); + + b.ToTable("Script", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ScriptedAlarm", b => + { + b.Property("ScriptedAlarmRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AlarmType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("HistorizeToAveva") + .HasColumnType("bit"); + + b.Property("MessageTemplate") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PredicateScriptId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Retain") + .HasColumnType("bit"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptedAlarmId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Severity") + .HasColumnType("int"); + + b.HasKey("ScriptedAlarmRowId"); + + b.HasIndex("PredicateScriptId") + .HasDatabaseName("IX_ScriptedAlarm_Script"); + + b.HasIndex("ScriptedAlarmId") + .IsUnique() + .HasDatabaseName("UX_ScriptedAlarm_LogicalId") + .HasFilter("[ScriptedAlarmId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "Name") + .IsUnique() + .HasDatabaseName("UX_ScriptedAlarm_EquipmentPath"); + + b.ToTable("ScriptedAlarm", null, t => + { + t.HasCheckConstraint("CK_ScriptedAlarm_AlarmType", "AlarmType IN ('AlarmCondition','LimitAlarm','OffNormalAlarm','DiscreteAlarm')"); + + t.HasCheckConstraint("CK_ScriptedAlarm_Severity_Range", "Severity BETWEEN 1 AND 1000"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", b => + { + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Enterprise") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2(3)"); + + b.Property("ModifiedBy") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("NodeCount") + .HasColumnType("tinyint"); + + b.Property("Notes") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("RedundancyMode") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Site") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("ClusterId"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("UX_ServerCluster_Name"); + + b.HasIndex("Site") + .HasDatabaseName("IX_ServerCluster_Site"); + + b.ToTable("ServerCluster", null, t => + { + t.HasCheckConstraint("CK_ServerCluster_RedundancyMode_NodeCount", "((NodeCount = 1 AND RedundancyMode = 'None') OR (NodeCount = 2 AND RedundancyMode IN ('Warm', 'Hot')))"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Tag", b => + { + b.Property("TagRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AccessLevel") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("DataType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("DeviceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PollGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("TagConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("TagId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("WriteIdempotent") + .HasColumnType("bit"); + + b.HasKey("TagRowId"); + + b.HasIndex("TagId") + .IsUnique() + .HasDatabaseName("UX_Tag_LogicalId") + .HasFilter("[TagId] IS NOT NULL"); + + b.HasIndex("DeviceId", "Name") + .IsUnique() + .HasDatabaseName("UX_Tag_Device_Name") + .HasFilter("[TagGroupId] IS NULL"); + + b.HasIndex("DeviceId", "TagGroupId") + .HasDatabaseName("IX_Tag_Device"); + + b.HasIndex("DeviceId", "TagGroupId", "Name") + .IsUnique() + .HasDatabaseName("UX_Tag_Group_Name") + .HasFilter("[TagGroupId] IS NOT NULL"); + + b.ToTable("Tag", null, t => + { + t.HasCheckConstraint("CK_Tag_TagConfig_IsJson", "ISJSON(TagConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.TagGroup", b => + { + b.Property("TagGroupRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DeviceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ParentTagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.Property("TagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("TagGroupRowId"); + + b.HasIndex("DeviceId") + .HasDatabaseName("IX_TagGroup_Device"); + + b.HasIndex("TagGroupId") + .IsUnique() + .HasDatabaseName("UX_TagGroup_LogicalId") + .HasFilter("[TagGroupId] IS NOT NULL"); + + b.HasIndex("DeviceId", "Name") + .IsUnique() + .HasDatabaseName("UX_TagGroup_DeviceRoot_Name") + .HasFilter("[ParentTagGroupId] IS NULL"); + + b.HasIndex("DeviceId", "ParentTagGroupId", "Name") + .IsUnique() + .HasDatabaseName("UX_TagGroup_Parent_Name") + .HasFilter("[ParentTagGroupId] IS NOT NULL"); + + b.ToTable("TagGroup", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsArea", b => + { + b.Property("UnsAreaRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("UnsAreaId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsAreaRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_UnsArea_Cluster"); + + b.HasIndex("UnsAreaId") + .IsUnique() + .HasDatabaseName("UX_UnsArea_LogicalId") + .HasFilter("[UnsAreaId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_UnsArea_ClusterName"); + + b.ToTable("UnsArea", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsLine", b => + { + b.Property("UnsLineRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("UnsAreaId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("UnsLineId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsLineRowId"); + + b.HasIndex("UnsAreaId") + .HasDatabaseName("IX_UnsLine_Area"); + + b.HasIndex("UnsLineId") + .IsUnique() + .HasDatabaseName("UX_UnsLine_LogicalId") + .HasFilter("[UnsLineId] IS NOT NULL"); + + b.HasIndex("UnsAreaId", "Name") + .IsUnique() + .HasDatabaseName("UX_UnsLine_AreaName"); + + b.ToTable("UnsLine", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsTagReference", b => + { + b.Property("UnsTagReferenceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DisplayNameOverride") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.Property("TagId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("UnsTagReferenceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsTagReferenceRowId"); + + b.HasIndex("EquipmentId") + .HasDatabaseName("IX_UnsTagReference_Equipment"); + + b.HasIndex("TagId") + .HasDatabaseName("IX_UnsTagReference_Tag"); + + b.HasIndex("UnsTagReferenceId") + .IsUnique() + .HasDatabaseName("UX_UnsTagReference_LogicalId") + .HasFilter("[UnsTagReferenceId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "TagId") + .IsUnique() + .HasDatabaseName("UX_UnsTagReference_Equip_Tag"); + + b.ToTable("UnsTagReference", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.VirtualTag", b => + { + b.Property("VirtualTagRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ChangeTriggered") + .HasColumnType("bit"); + + b.Property("DataType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Historize") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("TimerIntervalMs") + .HasColumnType("int"); + + b.Property("VirtualTagId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("VirtualTagRowId"); + + b.HasIndex("ScriptId") + .HasDatabaseName("IX_VirtualTag_Script"); + + b.HasIndex("VirtualTagId") + .IsUnique() + .HasDatabaseName("UX_VirtualTag_LogicalId") + .HasFilter("[VirtualTagId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "Name") + .IsUnique() + .HasDatabaseName("UX_VirtualTag_EquipmentPath"); + + b.ToTable("VirtualTag", null, t => + { + t.HasCheckConstraint("CK_VirtualTag_TimerInterval_Min", "TimerIntervalMs IS NULL OR TimerIntervalMs >= 50"); + + t.HasCheckConstraint("CK_VirtualTag_Trigger_AtLeastOne", "ChangeTriggered = 1 OR TimerIntervalMs IS NOT NULL"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany("Nodes") + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNodeCredential", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", "Node") + .WithMany("Credentials") + .HasForeignKey("NodeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Node"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverInstance", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.LdapGroupRoleMapping", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeDeploymentState", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Deployment", "Deployment") + .WithMany() + .HasForeignKey("DeploymentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", "Node") + .WithMany() + .HasForeignKey("NodeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Deployment"); + + b.Navigation("Node"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.RawFolder", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", null) + .WithMany("RawFolders") + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsArea", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.Navigation("Credentials"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", b => + { + b.Navigation("Nodes"); + + b.Navigation("RawFolders"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs new file mode 100644 index 00000000..58cf9a7e --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs @@ -0,0 +1,36 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations +{ + /// + /// Enables ALLOW_SNAPSHOT_ISOLATION on the config database so the deploy path can read every + /// table inside one SNAPSHOT transaction (DraftSnapshotFactory.FromConfigDbAsync): + /// the validator and the artifact composer then see one consistent state instead of two reads + /// an AdminUI save can land between. No schema change — the model snapshot is untouched. + /// + /// + /// ALTER DATABASE cannot run inside the user transaction the migrator wraps each migration in, + /// hence suppressTransaction: true. Turning the option ON does not need exclusive access (unlike + /// READ_COMMITTED_SNAPSHOT); it waits for in-flight transactions and then takes effect. Down + /// leaves it ON: a database that has run with it is not harmed by keeping it, and switching it OFF + /// would fail any deploy running at that instant. + /// + public partial class EnableSnapshotIsolation : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + "ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON;", + suppressTransaction: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + // Deliberately a no-op; see the class remarks. + } + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs index 0c0c47d9..fd7dc35a 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs @@ -66,6 +66,23 @@ public sealed class DraftSnapshot /// silently dropped when the permission trie is built, so the grant has to be caught here instead. public IReadOnlyList NodeAcls { get; init; } = []; + /// + /// Cluster rows. Not read by the row-level rules; carried so the deploy path validates the cluster + /// topology () and composes the artifact from + /// the SAME read as every other table — a second read between validate and compose is exactly how + /// an edit ships unvalidated. + /// + public IReadOnlyList ServerClusters { get; init; } = []; + + /// Cluster-node rows; see for why they ride the snapshot. + public IReadOnlyList ClusterNodes { get; init; } = []; + + /// + /// Data-plane API keys (peppered HMAC only, never a secret). Not validated; carried so the artifact + /// composer ships the keys a driver-only node authenticates against from the one consistent read. + /// + public IReadOnlyList ApiKeys { get; init; } = []; + /// Prior Equipment rows (any generation, same cluster) for stability checks. public IReadOnlyList PriorEquipment { get; init; } = []; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs index 3bfd0a5c..154d664c 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs @@ -1,3 +1,4 @@ +using System.Data; using Microsoft.EntityFrameworkCore; namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation; @@ -21,6 +22,16 @@ namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation; /// / are left null so the path-length rule uses its /// conservative upper bound. /// +/// +/// One consistent read. On a relational provider the tables are read inside a +/// transaction, so a /uns or /raw save that +/// lands mid-read cannot leave half the tables on the new state and half on the old. The deploy path +/// validates AND composes its artifact from this one snapshot, which is what makes "the artifact the +/// validator never saw" unrepresentable. SQL Server requires ALLOW_SNAPSHOT_ISOLATION ON +/// (migration EnableSnapshotIsolation); without it the first query fails with error 3952 and +/// the deploy is rejected rather than silently read unisolated. A non-relational provider (the +/// InMemory test provider) and a context already inside a transaction read without one. +/// /// public static class DraftSnapshotFactory { @@ -29,6 +40,23 @@ public static class DraftSnapshotFactory /// Cancellation token. /// A snapshot populated from the live DB, ready for . public static async Task FromConfigDbAsync(OtOpcUaConfigDbContext db, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(db); + + var ownsTransaction = db.Database.IsRelational() && db.Database.CurrentTransaction is null; + var tx = ownsTransaction + ? await db.Database.BeginTransactionAsync(IsolationLevel.Snapshot, ct).ConfigureAwait(false) + : null; + await using (tx) + { + var snapshot = await ReadAsync(db, ct).ConfigureAwait(false); + // Read-only: committing releases the snapshot's version-store reservation; nothing was written. + if (tx is not null) await tx.CommitAsync(ct).ConfigureAwait(false); + return snapshot; + } + } + + private static async Task ReadAsync(OtOpcUaConfigDbContext db, CancellationToken ct) => new DraftSnapshot { // GenerationId=0 and null Enterprise/Site are intentional conservative fallbacks at the @@ -36,6 +64,8 @@ public static class DraftSnapshotFactory // EquipmentUuid checks run in separate validator passes — no rule here reads these fields. GenerationId = 0, // generation model dropped; placeholder (no rule reads it) ClusterId = string.Empty, // global snapshot; rules compare entity ClusterId fields, not this + ServerClusters = await db.ServerClusters.AsNoTracking().ToListAsync(ct), + ClusterNodes = await db.ClusterNodes.AsNoTracking().ToListAsync(ct), // v3: Namespace entity retired — no namespace list to materialize. The Raw tree // (RawFolders/TagGroups) + UnsTagReferences feed the v3 rules (charset, tagname length, // effective-leaf uniqueness). @@ -53,6 +83,7 @@ public static class DraftSnapshotFactory Scripts = await db.Scripts.AsNoTracking().ToListAsync(ct), PollGroups = await db.PollGroups.AsNoTracking().ToListAsync(ct), NodeAcls = await db.NodeAcls.AsNoTracking().ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().ToListAsync(ct), PriorEquipment = [], // intentional: no prior-generation table to diff against at this level ActiveReservations = await db.ExternalIdReservations .AsNoTracking() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index d569fd08..4aabe3db 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -223,18 +223,14 @@ public sealed class AdminOperationsActor : ReceiveActor // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at // Hot/Warm and the constraint stays green, which would boot the runtime into an - // InvalidTopology band. ValidateClusterTopology catches that drift, but it isn't carried on - // the generation-versioned DraftSnapshot (the cluster/node rows aren't versioned), so it must - // be run separately here against the live rows. Read-only (AsNoTracking); errors fold into the - // same reject summary alongside the snapshot rules so a deploy failing either check is - // rejected with both sets of messages. ClusterId-ordered for a deterministic summary. - var clusters = await db.ServerClusters - .AsNoTracking() - .OrderBy(c => c.ClusterId) - .ToListAsync(); - var nodesByCluster = (await db.ClusterNodes - .AsNoTracking() - .ToListAsync()) + // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of + // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows. + // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing + // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary. + // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT + // transaction), not a second query that could observe a later edit. + var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList(); + var nodesByCluster = draft.ClusterNodes .GroupBy(n => n.ClusterId, StringComparer.Ordinal) .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); foreach (var cluster in clusters) @@ -310,8 +306,9 @@ public sealed class AdminOperationsActor : ReceiveActor try { const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) - var scriptSources = await db.Scripts.AsNoTracking().Select(s => s.SourceCode).ToListAsync(); - var compiled = scriptSources + // Projected from the snapshot already in hand — Script.SourceCode used to be read three times. + var compiled = draft.Scripts + .Select(s => s.SourceCode) .Where(src => !PassthroughScript.TryMatch(src, out _)) .Distinct(StringComparer.Ordinal) .Count(); @@ -331,7 +328,9 @@ public sealed class AdminOperationsActor : ReceiveActor advisory = null; } - var artifact = await ConfigComposer.SnapshotAndFlattenAsync(db); + // Composed from the validated snapshot, never from a second read: an edit that lands between + // validation and here is simply not in this artifact (R03:3.2). + var artifact = ConfigComposer.Compose(draft); var deploymentId = DeploymentId.NewId(); var revHash = RevisionHash.Parse(artifact.RevisionHash); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs index d73dd7b3..6f332143 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs @@ -1,9 +1,8 @@ using System.IO.Compression; using System.Security.Cryptography; using System.Text.Json; -using Microsoft.EntityFrameworkCore; using ZB.MOM.WW.OtOpcUa.Configuration; -using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; @@ -25,45 +24,65 @@ public static class ConfigComposer }; /// Reads the current configuration and returns a deterministic snapshot blob with revision hash. + /// + /// One consistent read (, a SNAPSHOT transaction on + /// SQL Server) handed to . The deploy path does not call this: it validates and + /// composes from the one it already holds, so nothing can land between the + /// validator's read and the composer's. This overload serves the drift probe and tests. + /// /// The configuration database context. /// The cancellation token for the operation. /// A deterministic containing the serialised snapshot bytes and its revision hash. public static async Task SnapshotAndFlattenAsync( OtOpcUaConfigDbContext db, CancellationToken ct = default) + => Compose(await DraftSnapshotFactory.FromConfigDbAsync(db, ct).ConfigureAwait(false)); + + /// + /// Serialises an already-read into the deterministic artifact. Pure over + /// its input: the same snapshot always yields the same bytes and hash, whatever the DB holds by now. + /// + /// The rows to compose — the SAME rows the validator was run over. + /// The compressed artifact blob and its revision hash. + public static ConfigArtifact Compose(DraftSnapshot snapshot) { + ArgumentNullException.ThrowIfNull(snapshot); + // v3 greenfield snapshot: the Raw tree (RawFolders → DriverInstances(+RawFolderId) → Devices(+DeviceConfig) // → TagGroups → Tags(DeviceId/TagGroupId, no EquipmentId/FolderPath/DriverInstanceId)) plus the UNS // projection (UnsAreas/UnsLines/Equipment/UnsTagReferences) and the retained VirtualTags/ScriptedAlarms/ // PollGroups/NodeAcls/Scripts. The Namespaces snapshot is RETIRED (the two OPC UA namespaces are implicit). // RevisionHash inputs follow the blob automatically — adding/removing a table shifts the SHA-256. - var snapshot = new + // Every collection is sorted by its natural key with an ORDINAL comparer so the hash is independent of + // both EF row order and the process culture (a culture-aware sort could order "a-1"/"A-1" differently + // on two central nodes and make the same config hash twice). + var ordered = new { - Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), - Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), - RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), - DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), - Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), - TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), - Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), - Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), - UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), - PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), - UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), - UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), - NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + Clusters = snapshot.ServerClusters.OrderBy(x => x.ClusterId, StringComparer.Ordinal).ToList(), + Nodes = snapshot.ClusterNodes.OrderBy(x => x.NodeId, StringComparer.Ordinal).ToList(), + RawFolders = snapshot.RawFolders.OrderBy(x => x.RawFolderId, StringComparer.Ordinal).ToList(), + DriverInstances = snapshot.DriverInstances.OrderBy(x => x.DriverInstanceId, StringComparer.Ordinal).ToList(), + Devices = snapshot.Devices.OrderBy(x => x.DeviceId, StringComparer.Ordinal).ToList(), + TagGroups = snapshot.TagGroups.OrderBy(x => x.TagGroupId, StringComparer.Ordinal).ToList(), + Equipment = snapshot.Equipment.OrderBy(x => x.EquipmentId, StringComparer.Ordinal).ToList(), + Tags = snapshot.Tags.OrderBy(x => x.TagId, StringComparer.Ordinal).ToList(), + UnsTagReferences = snapshot.UnsTagReferences.OrderBy(x => x.UnsTagReferenceId, StringComparer.Ordinal).ToList(), + PollGroups = snapshot.PollGroups.OrderBy(x => x.PollGroupId, StringComparer.Ordinal).ToList(), + UnsAreas = snapshot.UnsAreas.OrderBy(x => x.UnsAreaId, StringComparer.Ordinal).ToList(), + UnsLines = snapshot.UnsLines.OrderBy(x => x.UnsLineId, StringComparer.Ordinal).ToList(), + NodeAcls = snapshot.NodeAcls.OrderBy(x => x.NodeAclId, StringComparer.Ordinal).ToList(), // the data-plane credentials travel with the config, because a driver-only node // has no ConfigDb to read them from (per-cluster mesh Phase 4) and would otherwise deny every // UserName token. Only the peppered HMAC of each secret ships — never the secret — and the // pepper is node configuration that is deliberately NOT in the artifact, so a stolen blob // still cannot be turned into a working credential. This is also why revoking a key reaches a // driver-only node only on the next deployment; see the ApiKey entity's remarks. - ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), - Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), - VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), - ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + ApiKeys = snapshot.ApiKeys.OrderBy(x => x.ApiKeyId, StringComparer.Ordinal).ToList(), + Scripts = snapshot.Scripts.OrderBy(x => x.ScriptId, StringComparer.Ordinal).ToList(), + VirtualTags = snapshot.VirtualTags.OrderBy(x => x.VirtualTagId, StringComparer.Ordinal).ToList(), + ScriptedAlarms = snapshot.ScriptedAlarms.OrderBy(x => x.ScriptedAlarmId, StringComparer.Ordinal).ToList(), }; - return Pack(JsonSerializer.SerializeToUtf8Bytes(snapshot, JsonOptions)); + return Pack(JsonSerializer.SerializeToUtf8Bytes(ordered, JsonOptions)); } /// diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs new file mode 100644 index 00000000..aa16ead6 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs @@ -0,0 +1,117 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Tests; + +/// +/// R03:3.2 on the real provider: the migrated config database allows SNAPSHOT isolation (migration +/// EnableSnapshotIsolation) and reads under +/// it. The InMemory tests cannot see either — the provider has no transactions — so this is the one +/// place the deploy path's "one consistent read" is proven against SQL Server. +/// +[Trait("Category", "SchemaCompliance")] +[Collection(nameof(SchemaComplianceCollection))] +public sealed class DraftSnapshotFactorySqlServerTests +{ + private readonly SchemaComplianceFixture _fixture; + + /// Initializes a new instance of the class. + /// The migrated test database. + public DraftSnapshotFactorySqlServerTests(SchemaComplianceFixture fixture) => _fixture = fixture; + + /// The migration leaves the database with snapshot isolation allowed. + [Fact] + public void Migrated_database_allows_snapshot_isolation() + { + using var conn = _fixture.OpenConnection(); + using var cmd = new SqlCommand( + "SELECT snapshot_isolation_state_desc FROM sys.databases WHERE name = DB_NAME();", conn); + cmd.ExecuteScalar().ShouldBe("ON"); + } + + /// + /// A row committed by another connection while the snapshot read is in progress is NOT observed by + /// the tables read after it — every table reflects the instant the transaction started. Positive + /// control: a read started after the commit does see the row. + /// + [Fact] + public async Task FromConfigDb_reads_every_table_as_of_one_instant() + { + var clusterId = $"SNAP-{Guid.NewGuid():N}"[..24]; + var scriptId = $"late-{Guid.NewGuid():N}"; + await using (var seed = NewContext()) + { + seed.ServerClusters.Add(new ServerCluster + { + ClusterId = clusterId, Name = "snapshot-read", Enterprise = "zb", Site = "test", + NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + await seed.SaveChangesAsync(); + } + + try + { + var options = new DbContextOptionsBuilder() + .UseSqlServer(_fixture.ConnectionString) + .AddInterceptors(new LateEditInterceptor(_fixture.ConnectionString, clusterId, scriptId)) + .Options; + + await using (var db = new OtOpcUaConfigDbContext(options)) + { + var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(db); + // The interceptor committed a Script while ServerClusters (the FIRST table read) was being + // materialised. Scripts is read later in the same transaction and must not see it. + snapshot.ServerClusters.ShouldContain(c => c.ClusterId == clusterId); + snapshot.Scripts.ShouldNotContain(s => s.ScriptId == scriptId); + } + + await using (var verify = NewContext()) + { + // Positive control: the edit really landed, and a read started afterwards sees it. + (await verify.Scripts.AnyAsync(s => s.ScriptId == scriptId)).ShouldBeTrue(); + var later = await DraftSnapshotFactory.FromConfigDbAsync(verify); + later.Scripts.ShouldContain(s => s.ScriptId == scriptId); + } + } + finally + { + await using var cleanup = NewContext(); + await cleanup.Scripts.Where(s => s.ScriptId == scriptId).ExecuteDeleteAsync(); + await cleanup.ServerClusters.Where(c => c.ClusterId == clusterId).ExecuteDeleteAsync(); + } + } + + private OtOpcUaConfigDbContext NewContext() => new( + new DbContextOptionsBuilder().UseSqlServer(_fixture.ConnectionString).Options); + + /// + /// Commits a Script on a SEPARATE connection the first time the snapshot read materialises the + /// seeded ServerCluster row — i.e. mid-read, after the first table and before Scripts. + /// + private sealed class LateEditInterceptor(string connectionString, string clusterId, string scriptId) + : IMaterializationInterceptor + { + private int _fired; + + public object InitializedInstance(MaterializationInterceptionData materializationData, object entity) + { + if (entity is ServerCluster c && c.ClusterId == clusterId && Interlocked.Exchange(ref _fired, 1) == 0) + { + using var other = new OtOpcUaConfigDbContext( + new DbContextOptionsBuilder().UseSqlServer(connectionString).Options); + other.Scripts.Add(new Script + { + ScriptId = scriptId, Name = "late edit", SourceCode = "return 1;", SourceHash = "late-hash", + }); + other.SaveChanges(); + } + return entity; + } + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs index 9ad5806a..df1fad5a 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs @@ -103,6 +103,36 @@ public sealed class DraftSnapshotFactoryTests : IDisposable snapshot.ActiveReservations[0].ReleasedAt.ShouldBeNull(); } + /// + /// R03:3.2 — the deploy path validates AND composes from ONE snapshot, so the tables the composer + /// needs beyond the validator's set (clusters, nodes, API keys) must ride the same read. + /// + [Fact] + public async Task FromConfigDb_carries_clusters_nodes_and_api_keys_for_the_composer() + { + _db.ServerClusters.Add(new ServerCluster + { + ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", + NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + _db.ClusterNodes.Add(new ClusterNode + { + NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1", + ApplicationUri = "urn:OtOpcUa:site-a-1", Enabled = true, CreatedBy = "test", + }); + _db.ApiKeys.Add(new ApiKey + { + ApiKeyId = "k1", Name = "scada", SecretHash = "hash", Principal = "scadabridge", Enabled = true, + }); + await _db.SaveChangesAsync(); + + var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(_db); + + snapshot.ServerClusters.ShouldHaveSingleItem().ClusterId.ShouldBe("SITE-A"); + snapshot.ClusterNodes.ShouldHaveSingleItem().NodeId.ShouldBe("site-a-1:4053"); + snapshot.ApiKeys.ShouldHaveSingleItem().ApiKeyId.ShouldBe("k1"); + } + private void SeedEquipment(string equipmentId) { var uuid = Guid.NewGuid(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs new file mode 100644 index 00000000..aa5ded29 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs @@ -0,0 +1,133 @@ +using System.IO.Compression; +using System.Text; +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Storage; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:3.2 — StartDeployment used to read the whole config DB twice: once for the validator, +/// once for the composer. An edit landing between the two shipped an artifact the validator never saw. +/// The deploy now validates and composes ONE snapshot, so an edit that lands mid-deploy is either +/// validated (it was in the snapshot) or absent from the artifact (it was not) — never shipped unseen. +/// +public sealed class AdminOperationsActorSingleSnapshotTests : ControlPlaneActorTestBase +{ + private const string LateEquipmentId = "EQ-late-and-NOT-canonical"; + + /// + /// An Equipment with a non-canonical id (the validator rejects those with EquipmentIdNotDerived) + /// is committed by another context while the snapshot read is in progress, AFTER the Equipment table + /// has been read. The validator cannot see it, so the deploy is Accepted — and the artifact must not + /// carry it either. Positive control: the row really landed in the DB, and a deploy started after it + /// is Rejected on exactly that rule. + /// + [Fact] + public void Edit_landing_between_validate_and_compose_cannot_ship_unvalidated() + { + // One shared root: an InMemory store is scoped to the internal service provider, and a context + // carrying an interceptor builds its own provider — without the shared root the intercepting + // context would read an EMPTY database and the whole test would be vacuous. + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var plain = new RootedDbFactory(dbName, root, interceptor: null); + using (var db = plain.CreateDbContext()) + { + // A Tag to materialise — Tags is read AFTER Equipment by the snapshot factory, so an insert + // fired on the first Tag lands "between" the two tables of one read. + db.DriverInstances.Add(new DriverInstance + { + DriverInstanceId = "d", ClusterId = "", Name = "drv", DriverType = "Modbus", DriverConfig = "{}", + }); + db.Devices.Add(new Device { DeviceId = "dev", DriverInstanceId = "d", Name = "dev", DeviceConfig = "{}" }); + db.Tags.Add(new Tag + { + TagId = "tag-speed", DeviceId = "dev", Name = "speed", DataType = "Float", + AccessLevel = TagAccessLevel.Read, TagConfig = "{}", + }); + db.SaveChanges(); + } + + var injecting = new RootedDbFactory(dbName, root, new LateEquipmentInterceptor(plain)); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(injecting, coordinator.Ref, Enumerable.Empty())); + + actor.Tell(new StartDeployment("joe", CorrelationId.NewId())); + + var reply = ExpectMsg(PresenceBudget); + reply.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, reply.Message); + coordinator.ExpectMsg(PresenceBudget); + + using (var verify = plain.CreateDbContext()) + { + // The injection happened... + verify.Equipment.Any(e => e.EquipmentId == LateEquipmentId).ShouldBeTrue(); + // ...and the shipped artifact does not contain what the validator never saw. + var row = verify.Deployments.Single(); + Inflate(row.ArtifactBlob).ShouldNotContain(LateEquipmentId); + // The coordinator is a probe here, so seal the row by hand to let the next deploy through. + row.Status = DeploymentStatus.Sealed; + verify.SaveChanges(); + } + + // Positive control: now that the row is visible to a fresh read, the gate rejects it. + actor.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var second = ExpectMsg(PresenceBudget); + second.Outcome.ShouldBe(StartDeploymentOutcome.Rejected); + second.Message.ShouldNotBeNull(); + second.Message.ShouldContain("EquipmentIdNotDerived"); + } + + private static string Inflate(byte[] blob) + { + using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress); + using var reader = new StreamReader(gz, Encoding.UTF8); + return reader.ReadToEnd(); + } + + /// On the first Tag materialised, commits a non-canonical Equipment through ANOTHER context. + private sealed class LateEquipmentInterceptor(IDbContextFactory other) + : IMaterializationInterceptor + { + private int _fired; + + public object InitializedInstance(MaterializationInterceptionData materializationData, object entity) + { + if (entity is Tag && Interlocked.Exchange(ref _fired, 1) == 0) + { + using var db = other.CreateDbContext(); + db.Equipment.Add(new Equipment + { + EquipmentUuid = Guid.NewGuid(), EquipmentId = LateEquipmentId, Name = "late", + UnsLineId = "line-a", MachineCode = "m", + }); + db.SaveChanges(); + } + return entity; + } + } + + private sealed class RootedDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor? interceptor) + : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() + { + var builder = new DbContextOptionsBuilder().UseInMemoryDatabase(dbName, root); + if (interceptor is not null) builder.AddInterceptors(interceptor); + return new OtOpcUaConfigDbContext(builder.Options); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs index 4e431151..674ececd 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs @@ -222,6 +222,47 @@ public sealed class ConfigComposerTests : ControlPlaneActorTestBase withoutOverride.ResilienceConfig.ShouldBeNull("an unset override stays null ⇒ tier defaults at spawn"); } + /// + /// R03:3.2 — the deploy path composes from the it + /// validated rather than re-reading the DB. over the factory's + /// snapshot must therefore be byte-identical to the DB overload, or the drift probe (which still + /// uses the DB overload) would report every deployment as drifted. + /// + [Fact] + public async Task Compose_over_the_factory_snapshot_is_byte_identical_to_the_db_overload() + { + var f = NewInMemoryDbFactory(); + await using (var db = f.CreateDbContext()) + { + db.ServerClusters.Add(NewCluster("cluster-b")); + db.ServerClusters.Add(NewCluster("cluster-a")); + db.ApiKeys.Add(new ApiKey + { + ApiKeyId = "k1", Name = "scada", SecretHash = "h", Principal = "scadabridge", CreatedUtc = FixedTimestamp, + }); + db.Scripts.Add(new Script { ScriptId = "s-1", Name = "s", SourceCode = "return 1;", SourceHash = "x" }); + await db.SaveChangesAsync(); + } + + var viaDb = await ConfigComposer.SnapshotAndFlattenAsync(f.CreateDbContext()); + var snapshot = await Configuration.Validation.DraftSnapshotFactory.FromConfigDbAsync(f.CreateDbContext()); + var viaSnapshot = ConfigComposer.Compose(snapshot); + + viaSnapshot.RevisionHash.ShouldBe(viaDb.RevisionHash); + viaSnapshot.Blob.ShouldBe(viaDb.Blob); + // The composed content is the snapshot's, not something re-read: the keys and clusters are in it. + var json = Inflate(viaSnapshot.Blob); + json.ShouldContain("\"ApiKeyId\":\"k1\""); + json.IndexOf("cluster-a", StringComparison.Ordinal).ShouldBeLessThan(json.IndexOf("cluster-b", StringComparison.Ordinal)); + } + + private static string Inflate(byte[] blob) + { + using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress); + using var reader = new StreamReader(gz, Encoding.UTF8); + return reader.ReadToEnd(); + } + private static readonly DateTime FixedTimestamp = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); private static ServerCluster NewCluster(string id) => new() From eb9269c337d5ce074deb075b83bec1b54351b438 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:39:43 -0400 Subject: [PATCH 03/27] feat(scripting): ScriptRunGuard quarantines runaway scripts across every seam (E1) A timed-out script is abandoned, not stopped, and every seam re-ran it on the next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1). A shared ScriptRunGuard, keyed per script instance, now: - refuses a run while that script's previous orphan is still executing (Bad "previous evaluation still running" + otopcua.script.run_refused); - quarantines after 3 consecutive non-completions or an allocation-budget breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB): no further dispatch, Bad quality, one Error, otopcua.script.quarantined, cleared on source change / redeploy / reset; - starts the budget clock when the pool picks the run up, so starvation is no longer charged to the script (R07:P5), and cancels the evaluator token on timeout. Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async EvaluateAsync; the driver's pump and timer ticks now await the run behind a SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate holds state and annotates the condition BadInternalError). ScriptingOptions (Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts) carries the knobs; the Calculation driver config accepts the same two keys. Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with a healthy sibling as positive control), per-seam refusal/quarantine/reset tests. The S12 race test now uses one vtag id per loop: twelve concurrent evaluations of ONE id is a shape production never produces and the guard refuses by design. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ScriptedAlarmEngine.cs | 82 +++- .../IScriptQuarantineOwner.cs | 15 + .../ScriptRunGuard.cs | 387 ++++++++++++++++++ .../ScriptingOptions.cs | 42 ++ .../TimedScriptEvaluator.cs | 10 +- .../CalculationDriver.cs | 83 ++-- .../CalculationDriverFactoryExtensions.cs | 17 + .../CalculationDriverOptions.cs | 21 +- .../CalculationEvaluator.cs | 75 +++- .../CalculationTimerScheduler.cs | 20 +- .../Engines/RoslynVirtualTagEvaluator.cs | 56 ++- .../VirtualTags/VirtualTagHostActor.cs | 6 + .../ScriptedAlarmEngineRunGuardTests.cs | 105 +++++ .../ScriptRunGuardTests.cs | 306 ++++++++++++++ .../CalculationEvaluatorRunGuardTests.cs | 86 ++++ .../RoslynVirtualTagEvaluatorRunGuardTests.cs | 66 +++ .../RoslynVirtualTagEvaluatorTests.cs | 10 +- 17 files changed, 1313 insertions(+), 74 deletions(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index 4ccc3592..808935ae 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -73,6 +73,14 @@ public sealed class ScriptedAlarmEngine : IDisposable private readonly Func _clock; private readonly TimeSpan _scriptTimeout; + /// + /// Per-alarm run quarantine (keyed by alarm id, source identity = predicate source). Refuses a + /// predicate run while the alarm's previous (timed-out, abandoned) run is still executing and + /// quarantines after three consecutive non-completions or an allocation-budget breach — one orphan + /// per alarm instead of one per upstream push. Reset on every (a redeploy). + /// + private readonly ScriptRunGuard _guard; + // ConcurrentDictionary, not a plain Dictionary: every mutation happens under // _evalGate, but four read paths (GetState, GetAllStates, LoadedAlarmIds, // RunShelvingCheck) touch _alarms from arbitrary threads (Admin UI request @@ -204,13 +212,15 @@ public sealed class ScriptedAlarmEngine : IDisposable /// The logger for engine-level diagnostics. /// Optional function providing the current UTC time; defaults to DateTime.UtcNow. /// Optional timeout for script execution; defaults to the evaluator's default timeout. + /// Run-guard budget + quarantine threshold; null ⇒ defaults. public ScriptedAlarmEngine( ITagUpstreamSource upstream, IAlarmStateStore store, ScriptLoggerFactory loggerFactory, ILogger engineLogger, Func? clock = null, - TimeSpan? scriptTimeout = null) + TimeSpan? scriptTimeout = null, + ScriptingOptions? scripting = null) { _upstream = upstream ?? throw new ArgumentNullException(nameof(upstream)); _store = store ?? throw new ArgumentNullException(nameof(store)); @@ -218,8 +228,20 @@ public sealed class ScriptedAlarmEngine : IDisposable _engineLogger = engineLogger ?? throw new ArgumentNullException(nameof(engineLogger)); _clock = clock ?? (() => DateTime.UtcNow); _scriptTimeout = scriptTimeout ?? TimedScriptEvaluator.DefaultTimeout; + _guard = new ScriptRunGuard(scripting, _engineLogger); } + /// The run guard every predicate evaluation goes through (diagnostics / tests). + public ScriptRunGuard RunGuard => _guard; + + /// + /// OPC UA BadInternalError — the quality a condition is annotated with while its predicate + /// cannot be evaluated (timed out, refused because the previous run is still executing, or + /// quarantined). The condition's state is frozen; the quality says why it cannot be trusted. The + /// next completed evaluation restores the input-derived quality through the normal bucket tracking. + /// + private const uint StatusBadInternalError = 0x80020000u; + /// Raised for every emission the Part9StateMachine produces that the engine should publish. public event EventHandler? OnEvent; @@ -268,6 +290,9 @@ public sealed class ScriptedAlarmEngine : IDisposable // recompile this one. Skipping this is what made the earlier fix a // no-op in production. _compileCache.Clear(); + // A (re)load is a deploy: lift every run-guard quarantine (an edited predicate already lifts + // its own through the guard's source identity; this covers "redeployed to retry"). + _guard.ResetAll(); var compileFailures = new List(); foreach (var def in definitions) @@ -284,15 +309,17 @@ public sealed class ScriptedAlarmEngine : IDisposable // Route through CompiledScriptCache so the emitted assembly's // collectible ALC participates in publish-replace cleanup. + // The compiled script is run through _guard (timeout + quarantine) at evaluation time — + // the per-alarm TimedScriptEvaluator this used to wrap gave the timeout but could not + // see that the run it abandoned was still executing. var evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource); - var timed = new TimedScriptEvaluator(evaluator, _scriptTimeout); var logger = _loggerFactory.Create(def.AlarmId); var templateTokens = MessageTemplate.ExtractTokenPaths(def.MessageTemplate); var allInputs = new HashSet(extraction.Reads, StringComparer.Ordinal); foreach (var t in templateTokens) allInputs.Add(t); - _alarms[def.AlarmId] = new AlarmState(def, timed, extraction.Reads, templateTokens, logger, + _alarms[def.AlarmId] = new AlarmState(def, evaluator, extraction.Reads, templateTokens, logger, AlarmConditionState.Fresh(def.AlarmId, _clock())); foreach (var path in allInputs) @@ -767,22 +794,35 @@ public sealed class ScriptedAlarmEngine : IDisposable bool predicateTrue; try { - predicateTrue = await state.Evaluator.RunAsync(context, ct).ConfigureAwait(false); + predicateTrue = await _guard.RunAsync( + state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator, context, + _scriptTimeout, ct) + .ConfigureAwait(false); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } catch (ScriptTimeoutException tex) { - state.Logger.Warning("Alarm predicate timed out after {Timeout} — state unchanged", tex.Timeout); - return seed; + return PredicateDidNotComplete(state, seed, scratch, + $"timed out after {tex.Timeout.TotalMilliseconds:F0} ms", pendingEmissions); + } + catch (ScriptRunRefusedException ex) + { + return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions); + } + catch (ScriptQuarantinedException ex) + { + return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions); } catch (Exception ex) { + scratch.LastNonCompletion = null; state.Logger.Error(ex, "Alarm predicate threw — state unchanged"); return seed; } + scratch.LastNonCompletion = null; var result = Part9StateMachine.ApplyPredicate(seed, predicateTrue, nowUtc); var transition = result.Emission != EmissionKind.None @@ -802,6 +842,28 @@ public sealed class ScriptedAlarmEngine : IDisposable return result.State; } + /// + /// The predicate did not return (timed out, refused because the previous run is still executing, or + /// quarantined): hold the condition at , log the reason once per change of + /// reason (a quarantined alarm is re-attempted on every push and would otherwise log every time), + /// and annotate the condition out of band — the frozen state + /// cannot be trusted and the quality is how the operator sees that. The next completed evaluation + /// restores the input-derived quality through the normal bucket tracking. + /// + private AlarmConditionState PredicateDidNotComplete( + AlarmState state, AlarmConditionState seed, AlarmScratch scratch, string reason, + List pendingEmissions) + { + if (!string.Equals(scratch.LastNonCompletion, reason, StringComparison.Ordinal)) + { + scratch.LastNonCompletion = reason; + state.Logger.Warning("Alarm predicate did not complete ({Reason}) — state unchanged, quality Bad", reason); + } + if (TrackQualityBucket(state.Definition.AlarmId, StatusBadInternalError)) + pendingEmissions.Add(BuildQualityEmission(state, seed, StatusBadInternalError)); + return seed; + } + /// /// Refill in place from _valueCache, falling /// back to a synchronous ITagUpstreamSource.ReadTag for paths whose @@ -1201,7 +1263,7 @@ public sealed class ScriptedAlarmEngine : IDisposable private sealed record AlarmState( ScriptedAlarmDefinition Definition, - TimedScriptEvaluator Evaluator, + ScriptEvaluator Evaluator, IReadOnlySet Inputs, IReadOnlyList TemplateTokens, ILogger Logger, @@ -1227,6 +1289,10 @@ public sealed class ScriptedAlarmEngine : IDisposable /// Gets the predicate evaluation context. public AlarmPredicateContext Context { get; } + /// The reason the last evaluation did not complete (timeout / refused / quarantined), or null + /// after a completed run — the once-per-change log gate for PredicateDidNotComplete. + public string? LastNonCompletion { get; set; } + /// /// Initializes a new AlarmScratch with the specified inputs, logger, and clock. /// diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs new file mode 100644 index 00000000..3cf38fac --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs @@ -0,0 +1,15 @@ +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Implemented by a script seam that runs its scripts through a , so the +/// host that drives its deploy boundary can lift every quarantine on a redeploy (the guard's own +/// source-change detection covers an edited script; this covers "the operator redeployed to retry"). +/// +public interface IScriptQuarantineOwner +{ + /// Lift every quarantine and forget every consecutive-timeout count (). + void ResetQuarantines(); + + /// The script keys currently quarantined. + IReadOnlyCollection QuarantinedScripts { get; } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs new file mode 100644 index 00000000..8f1acefd --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs @@ -0,0 +1,387 @@ +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using Serilog; +using ZB.MOM.WW.OtOpcUa.Commons.Observability; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Per-script run quarantine shared by every script seam on the node. Wraps each evaluation of a +/// with the wall-clock timeout +/// used to provide, and adds what that wrapper +/// could not: knowledge of whether the script it abandoned is still running. +/// +/// +/// +/// The leak this closes. A timed-out script is abandoned, not stopped — a CPU-bound +/// while(true) keeps its thread-pool thread forever. Each seam re-evaluated on every trigger, +/// so a runaway script on a 1 Hz input leaked one pool thread per second until the pool starved and +/// unrelated work (Akka dispatchers, the OPC UA publish thread, gRPC) stalled. The guard keys every +/// run by script instance and: +/// +/// +/// Refuses to start a run while that script's previous run is still executing — +/// (), counted on +/// . One orphan per script, never one per trigger. +/// Quarantines after +/// consecutive non-completions (timeouts and still-running refusals both count; a run that +/// returns — a value or a throw — resets the count): no further dispatch, every attempt throws +/// , one Error log, one otopcua.script.quarantined +/// increment, one event. Lifted by a source change (the operator fixed the +/// script — a new source identity under the same key starts clean and the old orphan no longer blocks +/// it), by on redeploy, or by . +/// Quarantines on an allocation-budget breach: the run's own-thread allocation is sampled +/// with around the script delegate and compared +/// with . This is observation after the fact — a +/// single run that allocates enough to OOM the process is not stopped; that needs an out-of-process +/// runner. +/// +/// +/// The timeout clock starts when the script starts, not when it is queued. The run is pushed to +/// the pool (a CPU-bound script would otherwise execute inline and return an already-completed task +/// that no timeout can fire on), and the wait only begins once the pool has actually picked the run +/// up — so thread-pool starvation is charged to nobody's script. A caller-supplied cancellation wins +/// over the timeout (, never +/// ) so shutdown paths are not misread as timeouts. +/// +/// +/// Cancellation reaches the evaluator, not (yet) the script. On timeout the token handed to +/// is cancelled; a script can only observe +/// it once ScriptContext exposes it (follow-up — the context type lives in the Abstractions +/// project). +/// +/// +public sealed class ScriptRunGuard +{ + /// Failure reason a seam publishes when a run was refused because the previous one is still executing. + public const string PreviousRunStillRunningReason = "previous evaluation still running"; + + /// Prefix of every quarantine reason / message. + public const string QuarantinedReasonPrefix = "script quarantined: "; + + /// + /// Counter otopcua.script.quarantined (Prometheus: otopcua_script_quarantined): one + /// increment per script quarantined, tagged reason = timeouts | allocation. + /// + public static readonly Counter QuarantinedCounter = + OtOpcUaTelemetry.Meter.CreateCounter("otopcua.script.quarantined", unit: "{script}", + description: "Scripts quarantined by the run guard (reason: timeouts | allocation)."); + + /// + /// Counter otopcua.script.run_refused: runs refused because the script's previous run was + /// still executing. + /// + public static readonly Counter RefusedCounter = + OtOpcUaTelemetry.Meter.CreateCounter("otopcua.script.run_refused", unit: "{run}", + description: "Script runs refused because the previous run of the same script was still executing."); + + private readonly ConcurrentDictionary _keys = new(StringComparer.Ordinal); + private readonly long _maxAllocBytesPerRun; + private readonly int _quarantineThreshold; + private readonly ILogger _logger; + private int _inFlight; + private long _started; + private long _refused; + private long _quarantined; + + /// Initializes a new guard. + /// Budget + threshold; null ⇒ defaults. + /// Where the quarantine Error line goes; null ⇒ Serilog's global logger. + public ScriptRunGuard(ScriptingOptions? options = null, ILogger? logger = null) + { + options ??= new ScriptingOptions(); + _maxAllocBytesPerRun = options.MaxAllocBytesPerRun; + _quarantineThreshold = Math.Max(1, options.QuarantineAfterConsecutiveTimeouts); + _logger = logger ?? Log.Logger; + } + + /// Raised (on the thread that detected it) when a script is quarantined. + public event EventHandler? Quarantined; + + /// Runs currently executing on a pool thread — abandoned (timed-out) orphans included. + public int InFlightCount => Volatile.Read(ref _inFlight); + + /// Runs actually dispatched to the pool, lifetime. + public long StartedRunCount => Interlocked.Read(ref _started); + + /// Runs refused because the same script's previous run was still executing, lifetime. + public long RefusedWhileRunningCount => Interlocked.Read(ref _refused); + + /// Scripts quarantined, lifetime (a reset and re-quarantine counts twice). + public long QuarantinedCount => Interlocked.Read(ref _quarantined); + + /// The keys currently quarantined. + public IReadOnlyCollection QuarantinedKeys => + _keys.Where(kv => kv.Value.QuarantineReason is not null).Select(kv => kv.Key).ToArray(); + + /// Whether is quarantined, and why. + /// The script instance key. + /// The quarantine reason (prefixed ), or null. + /// true when quarantined. + public bool IsQuarantined(string key, out string? reason) + { + reason = _keys.TryGetValue(key, out var state) ? state.QuarantineReason : null; + return reason is not null; + } + + /// + /// Forget everything about : clears its quarantine and its consecutive count, and + /// abandons its in-flight slot (an orphan still running keeps counting on + /// until it finishes, but no longer blocks the key). + /// + /// The script instance key. + public void Reset(string key) => _keys.TryRemove(key, out _); + + /// + /// for every key — the redeploy / compile-cache-clear boundary. Orphans keep + /// counting on until they finish. + /// + public void ResetAll() => _keys.Clear(); + + /// + /// Run against under the guard. + /// + /// The script context type. + /// The script result type. + /// The script instance key (virtual-tag id, calc-tag RawPath, alarm id). + /// + /// The script source (or its hash). A change under the same key clears the key's quarantine and + /// abandons its orphan slot — the operator fixed the script. Null ⇒ not tracked. + /// + /// The compiled script. + /// The script's ctx. + /// Wall-clock budget, measured from the moment the pool starts the run. + /// Caller cancellation; wins over the timeout. + /// The script's result. + /// The key is (or just became) quarantined. + /// The key's previous run is still executing. + /// The run exceeded . + /// was cancelled. + public async Task RunAsync( + string key, + string? sourceIdentity, + ScriptEvaluator evaluator, + TContext context, + TimeSpan timeout, + CancellationToken ct = default) + where TContext : ScriptContext + { + ArgumentNullException.ThrowIfNull(key); + ArgumentNullException.ThrowIfNull(evaluator); + ArgumentNullException.ThrowIfNull(context); + if (timeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(timeout), "Timeout must be positive."); + + var state = Admit(key, sourceIdentity); + + // Slot acquisition is the whole mechanism: a second trigger landing while the previous run (orphan or + // not) still holds the slot is refused here, before anything is dispatched to the pool. + if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0) + { + Interlocked.Increment(ref _refused); + RefusedCounter.Add(1, new KeyValuePair("script", key)); + if (NoteNonCompletion(key, state, "refused: previous evaluation still running")) + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + throw new ScriptRunRefusedException(key); + } + + Interlocked.Increment(ref _inFlight); + Interlocked.Increment(ref _started); + + var slot = new RunSlot(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var runCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + var runToken = runCts.Token; + + // Task.Run deliberately takes NO token: a pre-cancelled token would skip the lambda, `started` would + // never complete, and the slot would never be released. The evaluator observes the token instead. + var runTask = Task.Run(async () => + { + started.TrySetResult(); + var before = GC.GetAllocatedBytesForCurrentThread(); + try + { + // ScriptEvaluator.RunAsync executes the compiled delegate synchronously and returns a completed + // task, so the after-sample is taken on the same thread, right after the script returned. + var pending = evaluator.RunAsync(context, runToken); + slot.AllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - before; + return await pending.ConfigureAwait(false); + } + catch + { + slot.AllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - before; + throw; + } + }); + + // Release the slot when the run REALLY finishes — which for an abandoned run is long after this + // method has thrown ScriptTimeoutException. Registered before the wait so it fires regardless of + // which way the wait below comes out. A late-finishing orphan that breached the allocation budget + // still quarantines its key here. + _ = runTask.ContinueWith(t => + { + Interlocked.Exchange(ref state.InFlight, 0); + Interlocked.Decrement(ref _inFlight); + runCts.Dispose(); + if (t.IsFaulted) _ = t.Exception; // observed below by the awaiter when it is still listening + if (t.IsCompletedSuccessfully && BreachedBudget(slot.AllocatedBytes)) + Quarantine(key, state, AllocationReason(slot.AllocatedBytes), "allocation"); + }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + + // The budget clock starts when the pool starts the run, not when we queued it (P5): pool starvation + // is a host problem and must not be reported as "script timed out". + await started.Task.WaitAsync(ct).ConfigureAwait(false); + + TResult result; + try + { + result = await runTask.WaitAsync(timeout, ct).ConfigureAwait(false); + } + catch (TimeoutException) + { + // The caller's cancel wins even when it lands a few µs after the timeout fired. + if (ct.IsCancellationRequested) throw new OperationCanceledException(ct); + runCts.Cancel(); + if (NoteNonCompletion(key, state, $"timed out after {timeout.TotalMilliseconds:F1} ms")) + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + throw new ScriptTimeoutException(timeout); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception) + { + // The script returned control (by throwing) — that is a completion for quarantine purposes. + Interlocked.Exchange(ref state.ConsecutiveNonCompletions, 0); + throw; + } + + Interlocked.Exchange(ref state.ConsecutiveNonCompletions, 0); + if (BreachedBudget(slot.AllocatedBytes)) + { + // The continuation above has already (or will have) quarantined the key; make the reason visible + // to THIS caller too, so the breaching run itself publishes Bad rather than a suspect value. + Quarantine(key, state, AllocationReason(slot.AllocatedBytes), "allocation"); + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + } + return result; + } + + /// Resolve the key's state, rolling it over when the source identity changed, and refuse a quarantined key. + private KeyState Admit(string key, string? sourceIdentity) + { + while (true) + { + var state = _keys.GetOrAdd(key, static (_, src) => new KeyState(src), sourceIdentity); + if (sourceIdentity is not null && state.SourceIdentity is not null + && !string.Equals(state.SourceIdentity, sourceIdentity, StringComparison.Ordinal)) + { + // The script changed under this key: start from a clean slate. The old state's orphan (if any) + // releases the GLOBAL in-flight count through its own continuation; nothing else references it. + _keys.TryUpdate(key, new KeyState(sourceIdentity), state); + continue; + } + if (sourceIdentity is not null && state.SourceIdentity is null) + { + state.SourceIdentity = sourceIdentity; + } + if (state.QuarantineReason is { } reason) + throw new ScriptQuarantinedException(key, reason); + return state; + } + } + + /// Record a non-completion; returns true when it tipped the key into quarantine. + private bool NoteNonCompletion(string key, KeyState state, string what) + { + var count = Interlocked.Increment(ref state.ConsecutiveNonCompletions); + if (count < _quarantineThreshold) return false; + Quarantine(key, state, + $"{count} consecutive evaluation(s) did not complete (last: {what}); " + + "the script is not evaluated again until its source changes or it is redeployed", + "timeouts"); + return true; + } + + private bool BreachedBudget(long allocatedBytes) => + _maxAllocBytesPerRun > 0 && allocatedBytes > _maxAllocBytesPerRun; + + private string AllocationReason(long allocatedBytes) => + $"one evaluation allocated {allocatedBytes / (1024.0 * 1024.0):F1} MB, over the " + + $"{_maxAllocBytesPerRun / (1024.0 * 1024.0):F0} MB budget (Scripting:MaxAllocBytesPerRun); " + + "the script is not evaluated again until its source changes or it is redeployed"; + + private void Quarantine(string key, KeyState state, string why, string metricReason) + { + var reason = QuarantinedReasonPrefix + why; + // First writer wins: a late orphan and the awaiter can both reach here for the same run. + if (Interlocked.CompareExchange(ref state.QuarantineReason, reason, null) is not null) return; + + Interlocked.Increment(ref _quarantined); + QuarantinedCounter.Add(1, + new KeyValuePair("script", key), + new KeyValuePair("reason", metricReason)); + _logger.Error("Script {ScriptKey} quarantined: {Reason}", key, why); + try { Quarantined?.Invoke(this, new ScriptQuarantinedEventArgs(key, reason)); } + catch (Exception ex) + { + _logger.Warning(ex, "ScriptRunGuard Quarantined subscriber threw for {ScriptKey}", key); + } + } + + private sealed class KeyState(string? sourceIdentity) + { + public string? SourceIdentity = sourceIdentity; + public int InFlight; + public int ConsecutiveNonCompletions; + public string? QuarantineReason; + } + + private sealed class RunSlot + { + public long AllocatedBytes; + } +} + +/// Carried by . +/// The quarantined script instance key. +/// Why, prefixed . +public sealed record ScriptQuarantinedEventArgs(string Key, string Reason); + +/// +/// Thrown by when the script's previous run is +/// still executing (an abandoned, timed-out run that never returned). Nothing was dispatched; the seam +/// publishes Bad with . +/// +public sealed class ScriptRunRefusedException : Exception +{ + /// The script instance key. + public string Key { get; } + + /// Initializes the exception for . + /// The script instance key. + public ScriptRunRefusedException(string key) : base(ScriptRunGuard.PreviousRunStillRunningReason) + { + Key = key; + } +} + +/// +/// Thrown by for a quarantined script: repeated +/// non-completion or an allocation-budget breach. Nothing is dispatched while quarantined; the seam +/// publishes Bad carrying the reason, which an operator clears by fixing the script (source change) or +/// redeploying. +/// +public sealed class ScriptQuarantinedException : Exception +{ + /// The script instance key. + public string Key { get; } + + /// Initializes the exception with the quarantine reason (already prefixed). + /// The script instance key. + /// The quarantine reason, prefixed . + public ScriptQuarantinedException(string key, string reason) : base(reason) + { + Key = key; + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs new file mode 100644 index 00000000..ba1399fa --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs @@ -0,0 +1,42 @@ +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Node-wide script runtime-safety knobs (the Scripting appsettings section), consumed by +/// . One instance is shared by every script seam on the node — the +/// virtual-tag evaluator, the Calculation evaluator and the scripted-alarm engine — so a single setting +/// bounds every authored script the same way. +/// +/// +/// +/// The in-process sandbox cannot pre-empt a script: a while(true) is abandoned on its pool +/// thread at the timeout, and an allocation is observed only after the run returns. These knobs bound +/// the damage — one orphan per script, no further runs until the source changes — not the +/// first occurrence. Pre-emptive CPU/memory budgeting needs an out-of-process script runner. +/// +/// +public sealed class ScriptingOptions +{ + /// The appsettings section this binds from. + public const string SectionName = "Scripting"; + + /// Default : 64 MB. + public const long DefaultMaxAllocBytesPerRun = 64L * 1024 * 1024; + + /// Default : 3. + public const int DefaultQuarantineAfterConsecutiveTimeouts = 3; + + /// + /// Managed bytes one script run may allocate on its own thread (sampled with + /// around the run) before the script is + /// quarantined. Zero or negative disables the budget. Default 64 MB. + /// + public long MaxAllocBytesPerRun { get; set; } = DefaultMaxAllocBytesPerRun; + + /// + /// Consecutive non-completions (a run that times out, or a run refused because the previous one is + /// still executing) after which the script is quarantined: no further runs are dispatched, every + /// evaluation reports Bad, and the quarantine lifts only on a source change, a redeploy or an explicit + /// reset. Default 3; values below 1 are treated as 1. + /// + public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts; +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs index 4adeccab..7a860436 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs @@ -19,10 +19,12 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// task continues running on a thread-pool thread until the Roslyn runtime /// returns. In the CPU-bound-infinite-loop case that's effectively "leaked" — /// the thread is tied up until the runtime decides to return, which it may -/// never do. Phase 7 plan Stream A.4 accepts this as a known trade-off; tighter -/// CPU budgeting would require an out-of-process script runner, which is a v3 -/// concern. In practice, the timeout + structured warning log surfaces the -/// offending script so the operator can fix it; the orphan thread is rare. +/// never do. This wrapper has no memory of that orphan, so re-running the same +/// script through it on every trigger leaks one more thread per trigger — which +/// is why the production seams (virtual tags, Calculation, scripted alarms) run +/// through instead: same timeout, plus a per-script +/// "previous run still executing" refusal and a quarantine. This class remains +/// for callers that evaluate once (tests, the authoring preview). /// /// /// Caller-supplied is honored — if the caller diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs index d556ccf6..dc0d8ba1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs @@ -92,7 +92,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // the timer ticks — the mutual exclusion _lock used to provide while it was held across the run, minus // the blocking. Taken ONLY by evaluation workers, so no host thread (mux callback, ReadAsync, // reinit) can ever wait on a script. Lock order is always _evalGate → _lock, never the reverse. - private readonly object _evalGate = new(); + // A SemaphoreSlim rather than a monitor because the holder AWAITS the script run (the pump is async — + // it no longer parks a second pool thread behind the one the script runs on, and the run-guard's + // budget clock starts only once the pool has picked the script up, so pool starvation is never + // reported as a script timeout). + private readonly SemaphoreSlim _evalGate = new(1, 1); + + // Cancels an awaited script run on Dispose (the only hard teardown — ShutdownAsync leaves an in-flight + // run to its RunTimeout, exactly as before, since the pump drops its result once _running is false). + private readonly CancellationTokenSource _evalCts = new(); // Change-triggered evaluations queued by OnDependencyValue and drained, in arrival order, by a single // pump task. Both fields are guarded by _lock; _pumpScheduled stays true until the pump observes an @@ -129,7 +137,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // _logger (ILogger) implements the non-generic ILogger, so it is passed straight // through — the evaluator's host-side diagnostics (compile errors, sandbox violations, execution // throws) land on the SAME logger the driver itself was constructed with, not a separate one. - _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout); + _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout, _options.Scripting); RebuildTagTable(_options); } @@ -225,6 +233,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba { RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [], RunTimeout = _options.RunTimeout, + Scripting = _options.Scripting, }; } } @@ -235,7 +244,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba _driverInstanceId); } } - return new CalculationDriverOptions { RawTags = _options.RawTags, RunTimeout = _options.RunTimeout }; + return new CalculationDriverOptions + { + RawTags = _options.RawTags, RunTimeout = _options.RunTimeout, Scripting = _options.Scripting, + }; } /// Re-bind the tag table from — shared by @@ -491,13 +503,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba { if (_pumpScheduled) return; _pumpScheduled = true; - _ = Task.Run(RunEvaluationPump); + _ = Task.Run(RunEvaluationPumpAsync); } - /// Drain the queued change-triggered evaluations, in arrival order, on a pool thread. At most - /// one pump runs at a time, so per-tag evaluation ordering is preserved exactly as it was when the - /// change path evaluated inline under the state lock. - private void RunEvaluationPump() + /// Drain the queued change-triggered evaluations, in arrival order. At most one pump runs at a + /// time, so per-tag evaluation ordering is preserved exactly as it was when the change path evaluated + /// inline under the state lock. Async: the pump AWAITS each script run instead of parking a pool thread + /// behind the one the script executes on (P5 — sync-over-async with a nested Task.Run manufactured + /// false timeouts under pool starvation). + private async Task RunEvaluationPumpAsync() { while (true) { @@ -515,39 +529,50 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // _evalGate (never _lock) is held across the run, so the pump and the timer ticks still // evaluate one at a time while no host thread can block behind a script. - lock (_evalGate) + await _evalGate.WaitAsync().ConfigureAwait(false); + try { - try - { - RunEvaluation(next); - } - catch (Exception ex) - { - // The evaluator never throws for script faults, so this is a defect path only — it - // must not kill the pump and strand every queued evaluation behind it. - _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed", - _driverInstanceId, next.Definition.RawPath); - } + await RunEvaluationAsync(next).ConfigureAwait(false); + } + catch (Exception ex) + { + // The evaluator never throws for script faults, so this is a defect path only — it + // must not kill the pump and strand every queued evaluation behind it. + _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed", + _driverInstanceId, next.Definition.RawPath); + } + finally + { + _evalGate.Release(); } } } - /// Timer-trigger entry point — prepares and evaluates a single calc tag on the scheduler's - /// pool thread (the scheduler's per-group in-flight guard is what bounds outstanding timer work). - private void EvaluateTagFromTimer(string rawPath) + /// Timer-trigger entry point — prepares and evaluates a single calc tag (the scheduler's + /// per-group in-flight guard is what bounds outstanding timer work; it stays set until this task + /// completes). + private async Task EvaluateTagFromTimerAsync(string rawPath) { if (!_running) return; PendingEvaluation? prepared; lock (_lock) prepared = PrepareEvaluation(rawPath, DateTime.UtcNow); if (prepared is null) return; - lock (_evalGate) RunEvaluation(prepared); + await _evalGate.WaitAsync().ConfigureAwait(false); + try + { + await RunEvaluationAsync(prepared).ConfigureAwait(false); + } + finally + { + _evalGate.Release(); + } } /// Run one prepared evaluation and publish the outcome. MUST be called under /// and NOT under : the script run is the whole point of the /// split, and the script log + fan-out both invoke subscriber code. /// The prepared evaluation. - private void RunEvaluation(PendingEvaluation pending) + private async Task RunEvaluationAsync(PendingEvaluation pending) { var def = pending.Definition; @@ -572,7 +597,8 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // Snapshots, not bare values: the script's ctx.GetTag(path).StatusCode / .SourceTimestampUtc are // then the input's real ones rather than a fabricated Good stamped at evaluation time. - var result = _evaluator.Evaluate(def.RawPath, def.ScriptSource, pending.Dependencies); + var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token) + .ConfigureAwait(false); string? failureReason = null; (object? Value, uint StatusCode)? toPublish; @@ -688,7 +714,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba { var timed = _tagsByRawPath.Values.Where(t => t.TimerInterval is not null).ToArray(); if (timed.Length == 0) return; - _timers = new CalculationTimerScheduler(EvaluateTagFromTimer, _logger); + _timers = new CalculationTimerScheduler(EvaluateTagFromTimerAsync, _logger); _timers.Start(timed); } @@ -723,6 +749,9 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba _running = false; StopTimers(); lock (_lock) _pending.Clear(); + // Cancel an awaited run first (the guard lets the caller's cancel win over the timeout), then drop + // the compiled scripts. The CTS is not disposed: a pump mid-await may still read its token. + try { _evalCts.Cancel(); } catch (ObjectDisposedException) { } _evaluator.Dispose(); } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs index ab10f06d..349a0cf1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs @@ -71,11 +71,22 @@ public static class CalculationDriverFactoryExtensions ?? throw new InvalidOperationException( $"Calculation driver config for '{driverInstanceId}' deserialised to null"); + // Run-guard knobs are optional and default to the node-wide ScriptingOptions defaults; only an + // explicitly authored key overrides one (so a config carrying neither binds Scripting = null). + ScriptingOptions? scripting = null; + if (dto.MaxAllocBytesPerRun is not null || dto.QuarantineAfterConsecutiveTimeouts is not null) + { + scripting = new ScriptingOptions(); + if (dto.MaxAllocBytesPerRun is { } alloc) scripting.MaxAllocBytesPerRun = alloc; + if (dto.QuarantineAfterConsecutiveTimeouts is { } n) scripting.QuarantineAfterConsecutiveTimeouts = n; + } + var options = new CalculationDriverOptions { RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [], RunTimeout = dto.RunTimeoutMs is { } ms && ms > 0 ? TimeSpan.FromMilliseconds(ms) : TimeSpan.FromSeconds(2), + Scripting = scripting, }; return new CalculationDriver( @@ -92,5 +103,11 @@ public static class CalculationDriverFactoryExtensions /// Optional per-script evaluation wall-clock budget override, in milliseconds (default 2000). public int? RunTimeoutMs { get; init; } + + /// Optional run-guard allocation budget per script run, in bytes (default 64 MB; ≤ 0 disables). + public long? MaxAllocBytesPerRun { get; init; } + + /// Optional run-guard quarantine threshold: consecutive non-completions before a tag's script stops being evaluated (default 3). + public int? QuarantineAfterConsecutiveTimeouts { get; init; } } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs index b6642c09..f00f50e4 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs @@ -1,20 +1,21 @@ using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; -/// -/// Bound options for a instance. The Calculation pseudo-driver has -/// no backend/endpoint config — its only input is the authored raw calc tags the deploy artifact -/// injects as the list (each carrying the tag's scriptId + resolved -/// scriptSource + trigger config inside its blob). -/// +/// Bound options for one instance. public sealed class CalculationDriverOptions { - /// The driver's authored raw calc tags (RawPath + TagConfig blob + WriteIdempotent), from - /// the deploy artifact. Each is mapped to a at construction. + /// The authored raw calc tags (RawPath + TagConfig JSON) this driver computes. public IReadOnlyList RawTags { get; init; } = Array.Empty(); - /// Per-script wall-clock evaluation budget handed to the . - /// Defaults to 2 seconds (parity with the VirtualTag evaluator). + /// Wall-clock budget per script run (driver-config runTimeoutMs); default 2 s. public TimeSpan RunTimeout { get; init; } = TimeSpan.FromSeconds(2); + + /// + /// Run-guard budget + quarantine threshold for this driver's scripts (driver-config + /// maxAllocBytesPerRun / quarantineAfterConsecutiveTimeouts). Null ⇒ the + /// defaults (64 MB, 3). + /// + public ScriptingOptions? Scripting { get; init; } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs index 249c6958..4296b993 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs @@ -37,8 +37,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// apply-boundary clear, which is what stops collectible AssemblyLoadContexts accreting across /// script edits. /// +/// +/// Run quarantine. Every compiled run goes through one keyed by +/// calc-tag RawPath with the script source as the identity: a run is refused while the tag's previous +/// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive +/// non-completions or an allocation-budget breach — both surface as a +/// the driver maps to Bad. The driver's evaluation pump awaits rather than +/// blocking a pool thread on the sync , +/// and the guard starts the budget clock only once the pool has picked the script up, so pool starvation +/// is never reported as a script timeout. +/// /// -public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable +public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineOwner, IDisposable { // Source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and Clear()/Dispose() // that dispose each evaluator's collectible AssemblyLoadContext. ClearCompiledScripts() — driven by the @@ -47,6 +57,7 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable private readonly ILogger _logger; private readonly SerilogLogger _scriptRoot; private readonly TimeSpan _runTimeout; + private readonly ScriptRunGuard _guard; private bool _disposed; /// Initializes a new . @@ -56,16 +67,22 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable /// implements this — instead of needing a second, separately-obtained logger instance. /// Root script logger; user ctx.Logger.* output flows through this to the Script-log page. /// Wall-clock budget per script run; defaults to 2 seconds (parity with the VT evaluator). + /// Run-guard budget + quarantine threshold; null ⇒ defaults. public CalculationEvaluator( ILogger logger, ScriptRootLogger scriptRoot, - TimeSpan? runTimeout = null) + TimeSpan? runTimeout = null, + ScriptingOptions? scripting = null) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger; _runTimeout = runTimeout ?? TimeSpan.FromSeconds(2); + _guard = new ScriptRunGuard(scripting, _scriptRoot); } + /// The run guard every compiled evaluation goes through (diagnostics / tests). + public ScriptRunGuard RunGuard => _guard; + /// /// Evaluate for the calc tag against /// the snapshot in (RawPath → last value). Never throws — script @@ -112,8 +129,25 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable /// The C# script source producing the tag's value. /// Read-only snapshot of dependency values keyed by RawPath. /// The evaluation outcome. + /// Blocks the calling thread for the run. The driver's pump uses ; + /// this overload remains for callers with no async context — chiefly tests. public VirtualTagEvalResult Evaluate( string calcTagId, string scriptSource, IReadOnlyDictionary dependencies) + => EvaluateAsync(calcTagId, scriptSource, dependencies).GetAwaiter().GetResult(); + + /// + /// The async form of : + /// the compiled script runs on the pool under the and the caller awaits + /// it instead of blocking a second pool thread for the duration. Never throws for script faults. + /// + /// Identity of the calc tag (bound to the script log). + /// The C# script source producing the tag's value. + /// Read-only snapshot of dependency values keyed by RawPath. + /// Caller cancellation (driver shutdown); wins over the run timeout. + /// The evaluation outcome. + public async Task EvaluateAsync( + string calcTagId, string scriptSource, IReadOnlyDictionary dependencies, + CancellationToken ct = default) { if (_disposed) return VirtualTagEvalResult.Failure("evaluator disposed"); if (string.IsNullOrWhiteSpace(scriptSource)) return VirtualTagEvalResult.Failure("empty expression"); @@ -180,11 +214,12 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable try { - // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't - // interrupt a CPU-bound/infinite-loop Roslyn script, so the timed wrapper is what bounds a - // runaway script to the wall-clock budget instead of wedging the driver's dispatch. - var timed = new TimedScriptEvaluator(evaluator, _runTimeout); - var raw = timed.RunAsync(context).GetAwaiter().GetResult(); + // Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by RawPath): a raw + // CancellationToken can't interrupt a CPU-bound/infinite-loop Roslyn script, so the wall-clock + // budget is what bounds a runaway script — and the guard is what keeps the next trigger from + // stacking another orphan behind it (one per tag, then quarantine). + var raw = await _guard.RunAsync(calcTagId, scriptSource, evaluator, context, _runTimeout, ct) + .ConfigureAwait(false); return VirtualTagEvalResult.Ok(raw); } catch (ScriptTimeoutException) @@ -192,6 +227,18 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable return VirtualTagEvalResult.Failure( $"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s"); } + catch (ScriptRunRefusedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } + catch (ScriptQuarantinedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + return VirtualTagEvalResult.Failure("evaluation cancelled (driver shutting down)"); + } catch (ObjectDisposedException) when (!_disposed && attempt == 0) { // A concurrent ClearCompiledScripts disposed the evaluator between our GetOrCompile and the @@ -212,7 +259,19 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable } /// - public void ClearCompiledScripts() => _cache.Clear(); + public void ClearCompiledScripts() + { + _cache.Clear(); + // The compile-cache clear is this evaluator's deploy boundary — a redeploy lifts every quarantine + // (an edited source already lifts its own through the guard's source identity). + _guard.ResetAll(); + } + + /// + public void ResetQuarantines() => _guard.ResetAll(); + + /// + public IReadOnlyCollection QuarantinedScripts => _guard.QuarantinedKeys; /// Disposes the evaluator, draining + disposing every cached compile's collectible ALC. public void Dispose() diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs index ef74da0e..694f7d8a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs @@ -7,11 +7,13 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// per distinct interval group (so the wire count stays low /// regardless of tag count) with a per-group in-flight flag that skips a tick when the prior tick for /// the same group is still running. Each tick invokes the driver's own evaluate -/// callback per tag in the group. +/// callback per tag in the group and AWAITS it — the in-flight flag is held across the awaited script +/// run, not just across the synchronous part of the callback, so a slow script still bounds its group +/// to one outstanding tick. /// internal sealed class CalculationTimerScheduler : IDisposable { - private readonly Action _evaluate; + private readonly Func _evaluate; private readonly ILogger _logger; private readonly List _timers = []; private readonly List _groups = []; @@ -20,9 +22,9 @@ internal sealed class CalculationTimerScheduler : IDisposable private bool _disposed; /// Initializes a new instance of the class. - /// Callback invoked with a calc tag's RawPath when its timer group ticks. + /// Callback invoked with a calc tag's RawPath when its timer group ticks; awaited. /// Logger for diagnostics. - public CalculationTimerScheduler(Action evaluate, ILogger logger) + public CalculationTimerScheduler(Func evaluate, ILogger logger) { _evaluate = evaluate ?? throw new ArgumentNullException(nameof(evaluate)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -69,12 +71,20 @@ internal sealed class CalculationTimerScheduler : IDisposable return; } + // The Timer callback is void; the tick itself is async so the awaited script runs do not pin this + // pool thread. TickAsync owns the in-flight flag and swallows everything — nothing can escape to the + // Timer (which would be an unobserved fault on a timer thread). + _ = TickAsync(ctx); + } + + private async Task TickAsync(TickGroup ctx) + { try { foreach (var path in ctx.Paths) { if (_cts.IsCancellationRequested) return; - try { _evaluate(path); } + try { await _evaluate(path).ConfigureAwait(false); } catch (Exception ex) { _logger.LogError(ex, "CalculationTimerScheduler evaluate failed for {Path}", path); } } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs index d77631ea..c6823c41 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs @@ -31,8 +31,17 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines; /// into the fresh cache for one generation; P7's expression-set gate means no-op redeploys no /// longer open this window at all. /// +/// +/// Run quarantine. Every compiled run goes through one keyed by +/// virtual-tag id with the expression as the source identity: a run is refused while the tag's previous +/// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive +/// non-completions or an allocation-budget breach. Both surface as +/// so VirtualTagActor publishes Bad exactly as it does for a timeout. A quarantine lifts when the +/// expression changes, on / (the deploy +/// boundary), never by itself. +/// /// -public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IDisposable +public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IScriptQuarantineOwner, IDisposable { // Purpose-built source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and // Clear()/Dispose() that dispose each evaluator's collectible AssemblyLoadContext. The apply-boundary @@ -42,22 +51,29 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac private readonly ILogger _logger; private readonly SerilogLogger _scriptRoot; private readonly TimeSpan _runTimeout; + private readonly ScriptRunGuard _guard; private bool _disposed; /// Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout. /// Logger for recording compilation and execution errors (host diagnostics). /// Root script logger; user ctx.Logger.* output flows through this to the Script-log page. /// Maximum execution time for each script; defaults to 2 seconds if not specified. + /// Run-guard budget + quarantine threshold (the Scripting section); null ⇒ defaults. public RoslynVirtualTagEvaluator( ILogger logger, ScriptRootLogger scriptRoot, - TimeSpan? runTimeout = null) + TimeSpan? runTimeout = null, + ScriptingOptions? scripting = null) { _logger = logger; _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger; _runTimeout = runTimeout ?? TimeSpan.FromSeconds(2); + _guard = new ScriptRunGuard(scripting, _scriptRoot); } + /// The run guard every compiled evaluation goes through (diagnostics / tests). + public ScriptRunGuard RunGuard => _guard; + // Value-only overload. Every dependency reaches the script stamped Good with an // evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot // overload instead; this one remains for callers that genuinely hold only @@ -150,13 +166,13 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac try { - // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't - // interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous - // CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when - // the wall-clock budget fires regardless of whether the inner task completes (the orphaned - // thread is the documented, accepted trade-off — TimedScriptEvaluator remarks). - var timed = new TimedScriptEvaluator(evaluator, _runTimeout); - var raw = timed.RunAsync(context).GetAwaiter().GetResult(); + // Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by virtual-tag id): a raw + // CancellationToken can't interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll + // it), so the wall-clock budget is what returns control — and the guard is what stops the + // abandoned run from being joined by another on the next trigger (one orphan per tag, then + // quarantine), which is the leak the old per-evaluation TimedScriptEvaluator allowed. + var raw = _guard.RunAsync(virtualTagId, expression, evaluator, context, _runTimeout) + .GetAwaiter().GetResult(); return VirtualTagEvalResult.Ok(raw); } catch (ScriptTimeoutException) @@ -164,6 +180,14 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac return VirtualTagEvalResult.Failure( $"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s"); } + catch (ScriptRunRefusedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } + catch (ScriptQuarantinedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } catch (ObjectDisposedException) when (!_disposed && attempt == 0) { // S12: the apply-boundary ClearCompiledScripts disposed the evaluator between our @@ -185,7 +209,19 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac } /// - public void ClearCompiledScripts() => _cache.Clear(); + public void ClearCompiledScripts() + { + _cache.Clear(); + // The compile-cache clear IS the deploy boundary for this evaluator: a redeploy lifts every quarantine + // (an edited expression already lifts its own through the guard's source identity). + _guard.ResetAll(); + } + + /// + public void ResetQuarantines() => _guard.ResetAll(); + + /// + public IReadOnlyCollection QuarantinedScripts => _guard.QuarantinedKeys; /// Disposes the evaluator and cleans up all cached script evaluators. public void Dispose() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs index d00dfb36..f97fa387 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs @@ -3,6 +3,7 @@ using Akka.Event; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Core.VirtualTags; using ZB.MOM.WW.OtOpcUa.OpcUaServer; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; @@ -112,6 +113,11 @@ public sealed class VirtualTagHostActor : ReceiveActor (_evaluator as IScriptCacheOwner)?.ClearCompiledScripts(); } + // Every apply is a deploy, and a deploy lifts the run guard's quarantines unconditionally — "the + // operator redeployed to retry" is one of the two documented ways out of a quarantine (the other, + // an edited expression, the guard detects on its own). Cheap on a no-op redeploy (an empty map). + (_evaluator as IScriptQuarantineOwner)?.ResetQuarantines(); + var desired = new HashSet(msg.Plans.Select(p => p.VirtualTagId), StringComparer.Ordinal); // Stop + forget children whose vtagId is no longer desired. Stopping the child triggers its diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs new file mode 100644 index 00000000..4074d0d9 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs @@ -0,0 +1,105 @@ +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; + +/// +/// runs every predicate through a keyed by +/// alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation +/// spawned another abandoned run); now the orphan is one, later pushes are refused and the alarm is +/// quarantined — frozen at its prior state and annotated Bad quality so the operator can see it — while a +/// sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine. +/// +[Trait("Category", "Unit")] +public sealed class ScriptedAlarmEngineRunGuardTests +{ + private const uint StatusBadInternalError = 0x80020000u; + + private const string SpinPredicate = + "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return false;"; + + private static ScriptedAlarmDefinition Alarm(string id, string predicate) => + new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, + Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); + + private static ScriptedAlarmEngine Build(FakeUpstream up, ScriptingOptions? scripting = null) + { + var logger = new LoggerConfiguration().CreateLogger(); + // 500 ms, not the 60 ms the guard's own tests use: the HEALTHY alarm's first run is JIT-cold and + // must finish inside the budget on a loaded CI box, or the positive control itself times out. The + // spin predicate outlives this by 12x, so the runaway alarm still cannot finish first. + return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger, + scriptTimeout: TimeSpan.FromMilliseconds(500), scripting: scripting); + } + + [Fact] + public async Task Runaway_predicate_leaks_one_orphan_is_quarantined_annotated_Bad_and_a_sibling_keeps_evaluating() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + using var eng = Build(up); + var events = new List(); + eng.OnEvent += (_, e) => { lock (events) events.Add(e); }; + + // The load itself evaluates once (startup recovery) — that is the first, abandoned run. + await eng.LoadAsync( + [Alarm("runaway", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"), + Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + + // Two more pushes: refused (orphan still running), then quarantined. + up.Push("Temp", 60); + await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= 1); + up.Push("Temp", 70); + await WaitForAsync(() => eng.RunGuard.IsQuarantined("runaway", out _)); + + eng.RunGuard.StartedRunCount.ShouldBe(1 + 3, "one dispatch for the runaway alarm, one per evaluation for the healthy one"); + eng.RunGuard.InFlightCount.ShouldBe(1); + + // The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving. + ScriptedAlarmEvent[] snapshot; + lock (events) snapshot = events.ToArray(); + var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray(); + quality.ShouldNotBeEmpty(); + quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError); + eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented"); + + // Positive control: the healthy sibling still transitions on the same engine. + up.Push("Temp", 150); + await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active); + } + + [Fact] + public async Task A_reload_lifts_the_quarantine() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + + await eng.LoadAsync([Alarm("a", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}")], + TestContext.Current.CancellationToken); + eng.RunGuard.IsQuarantined("a", out _).ShouldBeTrue(); + + // The operator fixed the predicate and redeployed. + await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + eng.RunGuard.IsQuarantined("a", out _).ShouldBeFalse(); + + up.Push("Temp", 150); + await WaitForAsync(() => eng.GetState("a")!.Active == AlarmActiveState.Active); + } + + private static async Task WaitForAsync(Func cond, int timeoutMs = 10_000) + { + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs); + while (DateTime.UtcNow < deadline) + { + if (cond()) return; + await Task.Delay(25); + } + throw new TimeoutException("Condition did not become true in time"); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs new file mode 100644 index 00000000..974e6a7c --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs @@ -0,0 +1,306 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; + +/// +/// — the per-script run quarantine shared by the virtual-tag evaluator, the +/// Calculation evaluator and the scripted-alarm engine. A timed-out script is abandoned, not stopped, so +/// before the guard every re-trigger of a runaway script leaked one more thread-pool thread; the guard +/// refuses to start a run while the previous one is still executing, quarantines a script after three +/// consecutive non-completions, and quarantines a script whose single run allocates past the budget. +/// +[Trait("Category", "Unit")] +public sealed class ScriptRunGuardTests +{ + /// + /// A CPU-bound script that outlives every timeout in this file but still ENDS (so the orphaned pool + /// thread it leaves behind frees itself after the test). System.DateTime is outside the sandbox + /// deny-list. + /// + private const string SpinScript = + """ + var end = System.DateTime.UtcNow.AddSeconds(6); + while (System.DateTime.UtcNow < end) { } + return 1; + """; + + private const string FastScript = "return 42;"; + + private static readonly TimeSpan ShortTimeout = TimeSpan.FromMilliseconds(60); + + private static ScriptEvaluator Compile(string source) => + ScriptEvaluator.Compile(source); + + private static Task Run(ScriptRunGuard guard, string key, string source, + ScriptEvaluator evaluator, TimeSpan? timeout = null, CancellationToken ct = default) => + guard.RunAsync(key, source, evaluator, new FakeScriptContext(), timeout ?? TimeSpan.FromSeconds(5), ct); + + [Fact] + public async Task Completed_run_returns_the_value_and_releases_the_in_flight_slot() + { + var guard = new ScriptRunGuard(); + using var fast = Compile(FastScript); + + var value = await Run(guard, "vt-1", FastScript, fast, ct: TestContext.Current.CancellationToken); + + value.ShouldBe(42); + guard.InFlightCount.ShouldBe(0); + guard.StartedRunCount.ShouldBe(1); + guard.IsQuarantined("vt-1", out _).ShouldBeFalse(); + } + + [Fact] + public async Task A_timed_out_run_throws_ScriptTimeoutException_and_keeps_the_slot_held_by_the_orphan() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + + await Should.ThrowAsync(() => + Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + // The abandoned script is still executing on its pool thread — the guard knows, the caller does not. + guard.InFlightCount.ShouldBe(1); + guard.StartedRunCount.ShouldBe(1); + } + + [Fact] + public async Task A_run_while_the_previous_orphan_is_still_executing_is_refused_without_starting_another() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + await Should.ThrowAsync(() => + Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + var ex = await Should.ThrowAsync(() => + Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + ex.Message.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + guard.StartedRunCount.ShouldBe(1, "the refusal must not dispatch a second Task.Run"); + guard.InFlightCount.ShouldBe(1); + guard.RefusedWhileRunningCount.ShouldBe(1); + } + + [Fact] + public async Task Three_consecutive_non_completions_quarantine_the_script_and_a_sibling_keeps_evaluating() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + // 1: times out, 2: refused (orphan still running), 3: refused → threshold reached → quarantined. + await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); + await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); + var quarantined = await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); + + quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + quarantined.Message.ShouldContain("3 consecutive"); + guard.IsQuarantined("bad", out var reason).ShouldBeTrue(); + reason.ShouldNotBeNull(); + guard.QuarantinedCount.ShouldBe(1); + + // A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run. + await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); + guard.StartedRunCount.ShouldBe(1); + + // Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key. + (await Run(guard, "good", FastScript, fast, ct: ct)).ShouldBe(42); + guard.IsQuarantined("good", out _).ShouldBeFalse(); + guard.StartedRunCount.ShouldBe(2); + } + + /// + /// The thread-leak regression. Pre-guard, a while(true) script on a 1 Hz trigger leaked one + /// pool thread per trigger — 30 orphans after 30 triggers, trending toward pool starvation. Now the + /// first trigger's orphan is the only one, every later trigger is refused or quarantined, and a healthy + /// sibling keeps evaluating throughout (the positive control an absence assertion needs). + /// + /// + /// The load-bearing assertions are on the guard's own bookkeeping ( + /// / ): moves with + /// every parallel test class on a loaded machine, so it is kept as a coarse secondary check only — + /// it must stay far below the 30 the leak would have produced. + /// + [Fact] + public async Task Runaway_script_leaves_at_most_one_orphan_after_30_triggers_while_a_sibling_keeps_evaluating() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + var threadsBefore = ThreadPool.ThreadCount; + + var healthyResults = new List(); + for (var trigger = 0; trigger < 30; trigger++) + { + try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); } + catch (ScriptTimeoutException) { } + catch (ScriptRunRefusedException) { } + catch (ScriptQuarantinedException) { } + + healthyResults.Add(await Run(guard, "healthy", FastScript, fast, ct: ct)); + } + + guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing"); + guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling"); + guard.IsQuarantined("runaway", out _).ShouldBeTrue(); + healthyResults.ShouldAllBe(v => v == 42); + healthyResults.Count.ShouldBe(30); + + var threadDelta = ThreadPool.ThreadCount - threadsBefore; + threadDelta.ShouldBeLessThan(15, "thirty leaked orphans would have grown the pool by ~30 threads"); + } + + [Fact] + public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); + await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); + await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("vt", out _).ShouldBeTrue(); + + // The operator fixed the script: a different source identity under the same key starts clean. The + // old orphan is a sunk cost — it keeps running, but it must not block the FIXED script. + (await Run(guard, "vt", FastScript, fast, ct: ct)).ShouldBe(42); + guard.IsQuarantined("vt", out _).ShouldBeFalse(); + guard.InFlightCount.ShouldBe(1, "the abandoned orphan is still counted until it finishes"); + } + + [Fact] + public async Task Reset_and_ResetAll_clear_a_quarantine() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + await Should.ThrowAsync(() => Run(guard, "a", SpinScript, spin, ShortTimeout, ct)); + await Should.ThrowAsync(() => Run(guard, "b", SpinScript, spin, ShortTimeout, ct)); + guard.QuarantinedKeys.ShouldBe(new[] { "a", "b" }, ignoreOrder: true); + + guard.Reset("a"); + guard.IsQuarantined("a", out _).ShouldBeFalse(); + guard.IsQuarantined("b", out _).ShouldBeTrue(); + + guard.ResetAll(); + guard.IsQuarantined("b", out _).ShouldBeFalse(); + // The key is usable again (a fresh slot: the orphan from the quarantined run no longer blocks it). + (await Run(guard, "b", FastScript, fast, ct: ct)).ShouldBe(42); + } + + [Fact] + public async Task A_run_that_allocates_past_the_budget_is_quarantined() + { + var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 1024 * 1024 }); + // Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the + // allocation as dead code (which is exactly what it does to the naive one-liner). + const string hungry = + "long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;"; + using var evaluator = Compile(hungry); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + var ex = await Should.ThrowAsync(() => Run(guard, "hog", hungry, evaluator, ct: ct)); + ex.Message.ShouldContain("allocat"); + guard.IsQuarantined("hog", out var reason).ShouldBeTrue(); + reason.ShouldNotBeNull(); + reason.ShouldContain("allocat"); + + // Never dispatched again, and the slot was released (the run DID complete). + await Should.ThrowAsync(() => Run(guard, "hog", hungry, evaluator, ct: ct)); + guard.StartedRunCount.ShouldBe(1); + guard.InFlightCount.ShouldBe(0); + + // Positive control: a modest allocation under the budget is fine. + (await Run(guard, "ok", FastScript, fast, ct: ct)).ShouldBe(42); + } + + [Fact] + public async Task A_disabled_allocation_budget_never_quarantines() + { + var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 0 }); + // Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the + // allocation as dead code (which is exactly what it does to the naive one-liner). + const string hungry = + "long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;"; + using var evaluator = Compile(hungry); + + (await Run(guard, "hog", hungry, evaluator, ct: TestContext.Current.CancellationToken)).ShouldBe(32 * (256 * 1024 + 1)); + guard.IsQuarantined("hog", out _).ShouldBeFalse(); + } + + [Fact] + public async Task A_script_that_throws_propagates_unwrapped_and_counts_as_a_completion() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 }); + const string throwing = "throw new System.InvalidOperationException(\"boom\");"; + using var thrower = Compile(throwing); + // A spin that outlives the timeout but finishes quickly, so the slot frees up between attempts. + const string briefSpin = + """ + var end = System.DateTime.UtcNow.AddMilliseconds(250); + while (System.DateTime.UtcNow < end) { } + return 1; + """; + using var spin = Compile(briefSpin); + var ct = TestContext.Current.CancellationToken; + + // timeout #1 + await Should.ThrowAsync(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct)); + await WaitForIdle(guard); + + // A throw is a completion — the script returned control — so the consecutive counter resets… + var ex = await Should.ThrowAsync(() => Run(guard, "k", throwing, thrower, ct: ct)); + ex.Message.ShouldBe("boom"); + guard.InFlightCount.ShouldBe(0); + + // …which is why this second timeout does NOT reach the threshold of 2. + await Should.ThrowAsync(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeFalse(); + await WaitForIdle(guard); + } + + [Fact] + public async Task Caller_cancellation_wins_over_the_timeout() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(80)); + + await Should.ThrowAsync(() => + Run(guard, "k", SpinScript, spin, TimeSpan.FromSeconds(5), cts.Token)); + guard.IsQuarantined("k", out _).ShouldBeFalse(); + } + + [Fact] + public async Task Quarantine_is_observable_through_the_event_and_the_counter() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + using var spin = Compile(SpinScript); + var seen = new List<(string Key, string Reason)>(); + guard.Quarantined += (_, e) => seen.Add((e.Key, e.Reason)); + + await Should.ThrowAsync(() => + Run(guard, "k", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + seen.ShouldHaveSingleItem(); + seen[0].Key.ShouldBe("k"); + seen[0].Reason.ShouldContain("consecutive"); + guard.QuarantinedCount.ShouldBe(1); + } + + private static async Task WaitForIdle(ScriptRunGuard guard) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (guard.InFlightCount != 0 && DateTime.UtcNow < deadline) + await Task.Delay(10); + guard.InFlightCount.ShouldBe(0, "the brief spin must have finished"); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs new file mode 100644 index 00000000..85943f16 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs @@ -0,0 +1,86 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests; + +/// +/// runs every compiled script through a +/// keyed by calc-tag RawPath. A timed-out run is abandoned on its pool thread; before the guard, every +/// later trigger of the same tag stacked another orphan behind it (one leaked thread per trigger). Now +/// the next trigger is refused ("previous evaluation still running"), the third consecutive +/// non-completion quarantines the tag, and a healthy sibling on the same evaluator keeps computing. +/// +public sealed class CalculationEvaluatorRunGuardTests +{ + private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger()); + + /// Outlives the 60 ms budget by a wide margin but still ends, so the orphan frees itself. + private const string SpinScript = + "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;"; + + private static readonly Dictionary NoDeps = new(); + + [Fact] + public async Task A_trigger_during_the_previous_orphan_is_refused_then_the_tag_is_quarantined_and_a_sibling_keeps_computing() + { + using var sut = new CalculationEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60)); + var ct = TestContext.Current.CancellationToken; + + var first = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct); + first.Success.ShouldBeFalse(); + first.Reason.ShouldNotBeNull(); + first.Reason.ShouldContain("timed out"); + + var second = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); + second.Success.ShouldBeFalse(); + second.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + + var third = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); + third.Success.ShouldBeFalse(); + third.Reason.ShouldNotBeNull(); + third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + sut.QuarantinedScripts.ShouldBe(new[] { "calc/loop" }); + + // Exactly one run was ever dispatched for the runaway tag — the leak is one orphan, not one per trigger. + sut.RunGuard.StartedRunCount.ShouldBe(1); + sut.RunGuard.InFlightCount.ShouldBe(1); + + // Positive control: a sibling tag on the same evaluator still computes. + var sibling = await sut.EvaluateAsync("calc/ok", "return 42;", Snap(NoDeps), ct); + sibling.Success.ShouldBeTrue(sibling.Reason); + sibling.Value.ShouldBe(42); + } + + [Fact] + public async Task ClearCompiledScripts_lifts_the_quarantine_as_a_redeploy_does() + { + using var sut = new CalculationEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60), + new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + var ct = TestContext.Current.CancellationToken; + + var r = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct); + r.Reason.ShouldNotBeNull(); + r.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + + sut.ClearCompiledScripts(); + + sut.QuarantinedScripts.ShouldBeEmpty(); + // The fixed script (new source) runs; the old orphan is a sunk cost that no longer blocks the tag. + var fixedRun = await sut.EvaluateAsync("calc/loop", "return 7;", Snap(NoDeps), ct); + fixedRun.Success.ShouldBeTrue(fixedRun.Reason); + fixedRun.Value.ShouldBe(7); + } + + private static Dictionary Snap(Dictionary deps) + { + var now = DateTime.UtcNow; + var d = new Dictionary(StringComparer.Ordinal); + foreach (var kv in deps) d[kv.Key] = new Core.Abstractions.DataValueSnapshot(kv.Value, 0u, now, now); + return d; + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs new file mode 100644 index 00000000..f0de4999 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Host.Engines; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// runs every compiled expression through a +/// keyed by virtual-tag id: a runaway expression leaks one orphaned pool +/// thread, not one per trigger, the tag is quarantined after three consecutive non-completions (surfacing +/// as a Failure the actor publishes Bad for), and the quarantine lifts at the deploy boundary. +/// +public sealed class RoslynVirtualTagEvaluatorRunGuardTests +{ + private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger()); + + private const string SpinScript = + "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;"; + + private static readonly Dictionary NoDeps = new(); + + [Fact] + public void Runaway_expression_is_refused_then_quarantined_and_a_sibling_keeps_evaluating() + { + using var sut = new RoslynVirtualTagEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60)); + + var first = sut.Evaluate("vt-loop", SpinScript, NoDeps); + first.Success.ShouldBeFalse(); + first.Reason.ShouldNotBeNull(); + first.Reason.ShouldContain("timed out"); + + sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + + var third = sut.Evaluate("vt-loop", SpinScript, NoDeps); + third.Reason.ShouldNotBeNull(); + third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + sut.QuarantinedScripts.ShouldBe(new[] { "vt-loop" }); + sut.RunGuard.StartedRunCount.ShouldBe(1, "one orphan, not one per trigger"); + + var sibling = sut.Evaluate("vt-ok", "return 42;", NoDeps); + sibling.Success.ShouldBeTrue(sibling.Reason); + sibling.Value.ShouldBe(42); + } + + [Fact] + public void ResetQuarantines_lifts_the_quarantine_as_a_redeploy_does() + { + using var sut = new RoslynVirtualTagEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60), + new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + + sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + sut.QuarantinedScripts.ShouldNotBeEmpty(); + + sut.ResetQuarantines(); + + sut.QuarantinedScripts.ShouldBeEmpty(); + var fixedRun = sut.Evaluate("vt-loop", "return 7;", NoDeps); + fixedRun.Success.ShouldBeTrue(fixedRun.Reason); + fixedRun.Value.ShouldBe(7); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs index 004b7165..47ac617a 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs @@ -489,12 +489,18 @@ public sealed class RoslynVirtualTagEvaluatorTests // most evaluations then hit the warm cache instead of recompiling. Neither the cadence nor the // iteration count is a correctness knob any more — the assertions below hold for any number of // clears landing inside a single evaluation, and for any number of evaluations completed. - var evalLoops = Enumerable.Range(0, loopCount).Select(_ => Task.Run(() => + // One virtual-tag id PER LOOP, sharing the expression. The race under test lives in the compile + // cache, which is keyed by source, so twelve ids on one expression contend for exactly the entry + // twelve copies of one id did — while twelve concurrent evaluations of ONE id is a shape production + // never produces (one VirtualTagActor per id serialises them) and exactly the shape the per-tag + // ScriptRunGuard refuses ("previous evaluation still running", then quarantine). + var evalLoops = Enumerable.Range(0, loopCount).Select(loop => Task.Run(() => { + var vtagId = $"vt-race-{loop}"; var done = 0; for (var i = 0; i < MaxIterationsPerLoop && Environment.TickCount64 < stopAtMs; i++) { - var r = sut.Evaluate("vt-race", expr, new Dictionary { ["a"] = i }); + var r = sut.Evaluate(vtagId, expr, new Dictionary { ["a"] = i }); done++; if (!r.Success) { From ae84253773481894b413052f3efcb1ad923f4e90 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:41:35 -0400 Subject: [PATCH 04/27] fix(controlplane): keep the admin singleton's mailbox free during Test Connect and deploy compose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReceiveAsync suspended the mailbox for the whole of every handler, so one Test Connect against a black-holed PLC (60 s clamp) or one large deploy compose stalled every other admin operation fleet-wide — AdminUI alarm acks/shelves, driver restarts, the next deploy. - TestDriverConnect runs the probe on the thread pool and pipes a TestDriverConnectResult straight to the requester (every path yields a result, never Status.Failure). - StartDeployment serialises only the in-flight check (DB row + an actor-local compose-in-flight flag, because the Deployment row is written at the END of the compose); the snapshot read, validation, gzip and row write run off-actor and return as DeployComposed / DeployComposeFailed, where the coordinator is told and the requester answered. Tests: an alarm ack completes in < 1 s while a 60 s probe is pending and while a compose is parked on a gated DbContext (positive controls: both still complete afterwards); a second StartDeployment during an in-flight compose is refused. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdminOperations/AdminOperationsActor.cs | 454 +++++++++++------- .../AdminOperationsActorMailboxTests.cs | 159 ++++++ 2 files changed, 432 insertions(+), 181 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index 4aabe3db..8b1c9ddb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -36,6 +36,9 @@ public sealed class AdminOperationsActor : ReceiveActor private readonly ILoggingAdapter _log = Context.GetLogger(); + /// The StartDeployment whose compose is running off-actor, or null. See . + private StartDeployment? _composeInFlight; + /// Creates actor props for the admin operations actor. /// Factory for creating config database contexts. /// Reference to the deployment coordinator actor. @@ -74,8 +77,12 @@ public sealed class AdminOperationsActor : ReceiveActor _tagConfigValidationMode = tagConfigValidationMode; _meshRouter = meshRouter; - ReceiveAsync(HandleStartDeploymentAsync); - ReceiveAsync(HandleTestDriverConnectAsync); + // Synchronous receives: the compose and the probe run off-actor and come back as messages, so + // the mailbox is never suspended for the length of a DB snapshot or a PLC connect timeout. + Receive(HandleStartDeployment); + Receive(HandleDeployComposed); + Receive(HandleDeployComposeFailed); + Receive(HandleTestDriverConnect); ReceiveAsync(HandleRestartDriverAsync); ReceiveAsync(HandleReconnectDriverAsync); Receive(HandleAcknowledgeAlarm); @@ -181,19 +188,42 @@ public sealed class AdminOperationsActor : ReceiveActor } } - private async Task HandleStartDeploymentAsync(StartDeployment msg) + /// + /// Serialised part of StartDeployment: the in-flight check. Everything after it — the + /// snapshot read, validation, gzip and the row write — runs on the thread pool and comes back as a + /// / message, so a large compose no + /// longer suspends the mailbox (R03:3.1) and alarm acks keep flowing while it runs. + /// + /// + /// is what keeps the check correct once the compose is off-actor: the + /// Deployment row is written at the END of the compose, so a second request arriving mid-compose + /// would pass a DB-only check and two artifacts would race for one coordinator. The flag is cleared + /// when the outcome message lands, on both branches. + /// + private void HandleStartDeployment(StartDeployment msg) { var replyTo = Sender; + if (_composeInFlight is { } inFlightRequest) + { + replyTo.Tell(new StartDeploymentResult( + StartDeploymentOutcome.AnotherDeploymentInFlight, + DeploymentId: null, + RevisionHash: null, + Message: $"A deployment requested by {inFlightRequest.CreatedBy} is still being composed.", + msg.CorrelationId)); + return; + } + try { - await using var db = await _dbFactory.CreateDbContextAsync(); - // Refuse if any deployment is already in flight — keeps the coordinator's state - // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. - var inflight = await db.Deployments + // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. One indexed + // projection (IX_Deployment_Status), synchronous on purpose: it IS the serialised part. + using var db = _dbFactory.CreateDbContext(); + var inflight = db.Deployments .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) .Select(d => d.DeploymentId) - .FirstOrDefaultAsync(); + .FirstOrDefault(); if (inflight != Guid.Empty) { replyTo.Tell(new StartDeploymentResult( @@ -204,172 +234,6 @@ public sealed class AdminOperationsActor : ReceiveActor msg.CorrelationId)); return; } - - // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject - // on ANY validation error. The config is now canonical — the company overlay loader emits - // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId - // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions, - // …) gates the deploy. A green build in this repo does not prove the config is valid; this - // is the last guard before a bad address space (or a non-derived EquipmentId) ships. - // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can - // (rarely) be a false positive if an external-id reservation release has not yet been - // committed/visible when the snapshot is read — operators seeing a spurious one should - // check ExternalIdReservation state before re-submitting. - var draft = await DraftSnapshotFactory.FromConfigDbAsync(db); - var errors = DraftValidator.Validate(draft).ToList(); - - // Cluster-topology guard. The SQL - // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode) - // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an - // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at - // Hot/Warm and the constraint stays green, which would boot the runtime into an - // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of - // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows. - // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing - // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary. - // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT - // transaction), not a second query that could observe a later edit. - var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList(); - var nodesByCluster = draft.ClusterNodes - .GroupBy(n => n.ClusterId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); - foreach (var cluster in clusters) - { - var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns) - ? (IReadOnlyList)ns - : []; - errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes)); - } - - // Deploy-time TagConfig strictness inspection over equipment tags. The - // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would - // REJECT — a present-but-invalid enum value, an absent or malformed address field, a - // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads - // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum; - // it now rejects the tag, so these warnings describe rejections.) In Warn mode - // (default) these are non-blocking (logged + appended to the accepted result Message); in Error - // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed. - // Running servers are untouched either way — the gate only sees re-deploys. - // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId / - // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type. - var driverTypeById = draft.DriverInstances - .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal); - var driverByDevice = draft.Devices - .GroupBy(d => d.DeviceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal); - var deviceConfigById = draft.Devices - .GroupBy(d => d.DeviceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal); - var tagConfigWarnings = new List<(string TagId, string Text)>(); - foreach (var t in draft.Tags) - { - if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue; - if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue; - foreach (var w in EquipmentTagConfigInspector.Inspect( - driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId))) - tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}")); - } - if (tagConfigWarnings.Count > 0) - { - if (_tagConfigValidationMode == TagConfigValidationMode.Error) - errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId))); - else - foreach (var x in tagConfigWarnings) - _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text); - } - - if (errors.Count > 0) - { - var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}")); - _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}", - errors.Count, summary); - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Rejected, - DeploymentId: null, - RevisionHash: null, - Message: summary, - msg.CorrelationId)); - return; - } - - // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough) - // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc - // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct - // sources are counted (the compile cache keys on source, so duplicates collapse to one unit). - // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard - // reject; this advisory rides in the Accepted Message so the UI can show it. - // advisory is declared outside the inner try so the Accepted reply below can still read it even - // when the guardrail query throws (e.g. transient SQL). A failed estimate must NEVER block a - // valid deploy — the outer catch is reserved for genuine seal/save failures. - string? advisory = null; - try - { - const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) - // Projected from the snapshot already in hand — Script.SourceCode used to be read three times. - var compiled = draft.Scripts - .Select(s => s.SourceCode) - .Where(src => !PassthroughScript.TryMatch(src, out _)) - .Distinct(StringComparer.Ordinal) - .Count(); - if (compiled > 0) - { - var estMiB = compiled * PerScriptMiB; - _log.Warning( - "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it", - compiled, estMiB); - advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)"; - } - } - catch (Exception ex) - { - // Guardrail is advisory-only — a failed estimate must never block a valid deploy. - _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped"); - advisory = null; - } - - // Composed from the validated snapshot, never from a second read: an edit that lands between - // validation and here is simply not in this artifact (R03:3.2). - var artifact = ConfigComposer.Compose(draft); - var deploymentId = DeploymentId.NewId(); - var revHash = RevisionHash.Parse(artifact.RevisionHash); - - db.Deployments.Add(new Deployment - { - DeploymentId = deploymentId.Value, - RevisionHash = artifact.RevisionHash, - Status = DeploymentStatus.Dispatching, - CreatedBy = msg.CreatedBy, - ArtifactBlob = artifact.Blob, - }); - - // Marker ConfigEdit row so the audit timeline shows the deployment snapshot. - db.ConfigEdits.Add(new ConfigEdit - { - EntityType = "Deployment", - EntityId = deploymentId.Value, - FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}", - EditedBy = msg.CreatedBy, - SourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown", - }); - - await db.SaveChangesAsync(); - - _coordinator.Tell(new DispatchDeployment(deploymentId, revHash, msg.CorrelationId)); - - // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so - // the operator sees the non-blocking strictness findings on a successful deploy. - var messageParts = new List(); - if (advisory is not null) messageParts.Add(advisory); - messageParts.AddRange(tagConfigWarnings.Select(x => x.Text)); - var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null; - - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Accepted, - deploymentId, - revHash, - Message: acceptedMessage, - msg.CorrelationId)); } catch (Exception ex) { @@ -380,10 +244,233 @@ public sealed class AdminOperationsActor : ReceiveActor RevisionHash: null, Message: ex.Message, msg.CorrelationId)); + return; } + + _composeInFlight = msg; + // Captured here: Context/Sender are not available on the thread-pool thread. + var sourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown"; + var self = Self; + Task.Run(() => ComposeDeploymentAsync(msg, sourceNode)) + .PipeTo(self, + success: composed => new DeployComposed(msg, replyTo, composed.Result, composed.Dispatch), + failure: ex => new DeployComposeFailed(msg, replyTo, ex)); } - private async Task HandleTestDriverConnectAsync(TestDriverConnect msg) + private void HandleDeployComposed(DeployComposed msg) + { + _composeInFlight = null; + // The row is committed (the compose task completed), so the coordinator can be told now; it reads + // the row back by id when it seeds the expected-ack set. + if (msg.Dispatch is { } dispatch) _coordinator.Tell(dispatch); + msg.ReplyTo.Tell(msg.Result); + } + + private void HandleDeployComposeFailed(DeployComposeFailed msg) + { + _composeInFlight = null; + _log.Error(msg.Error, "StartDeployment failed for {CreatedBy}", msg.Request.CreatedBy); + msg.ReplyTo.Tell(new StartDeploymentResult( + StartDeploymentOutcome.Rejected, + DeploymentId: null, + RevisionHash: null, + Message: msg.Error.Message, + msg.Request.CorrelationId)); + } + + /// Outcome of the off-actor compose: the reply to send, and the dispatch to issue when Accepted. + private sealed record ComposeOutcome(StartDeploymentResult Result, DispatchDeployment? Dispatch); + + private sealed record DeployComposed( + StartDeployment Request, IActorRef ReplyTo, StartDeploymentResult Result, DispatchDeployment? Dispatch); + + private sealed record DeployComposeFailed(StartDeployment Request, IActorRef ReplyTo, Exception Error); + + /// + /// The deploy compose, off the actor thread. Touches no actor state: everything it needs is in + /// , and the readonly fields captured at + /// construction (, , — + /// Akka's logging adapter is thread-safe). + /// + private async Task ComposeDeploymentAsync(StartDeployment msg, string sourceNode) + { + await using var db = await _dbFactory.CreateDbContextAsync(); + + // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject + // on ANY validation error. The config is now canonical — the company overlay loader emits + // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId + // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions, + // …) gates the deploy. A green build in this repo does not prove the config is valid; this + // is the last guard before a bad address space (or a non-derived EquipmentId) ships. + // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can + // (rarely) be a false positive if an external-id reservation release has not yet been + // committed/visible when the snapshot is read — operators seeing a spurious one should + // check ExternalIdReservation state before re-submitting. + var draft = await DraftSnapshotFactory.FromConfigDbAsync(db); + var errors = DraftValidator.Validate(draft).ToList(); + + // Cluster-topology guard. The SQL + // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode) + // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an + // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at + // Hot/Warm and the constraint stays green, which would boot the runtime into an + // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of + // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows. + // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing + // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary. + // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT + // transaction), not a second query that could observe a later edit. + var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList(); + var nodesByCluster = draft.ClusterNodes + .GroupBy(n => n.ClusterId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); + foreach (var cluster in clusters) + { + var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns) + ? (IReadOnlyList)ns + : []; + errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes)); + } + + // Deploy-time TagConfig strictness inspection over equipment tags. The + // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would + // REJECT — a present-but-invalid enum value, an absent or malformed address field, a + // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads + // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum; + // it now rejects the tag, so these warnings describe rejections.) In Warn mode + // (default) these are non-blocking (logged + appended to the accepted result Message); in Error + // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed. + // Running servers are untouched either way — the gate only sees re-deploys. + // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId / + // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type. + var driverTypeById = draft.DriverInstances + .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal); + var driverByDevice = draft.Devices + .GroupBy(d => d.DeviceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal); + var deviceConfigById = draft.Devices + .GroupBy(d => d.DeviceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal); + var tagConfigWarnings = new List<(string TagId, string Text)>(); + foreach (var t in draft.Tags) + { + if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue; + if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue; + foreach (var w in EquipmentTagConfigInspector.Inspect( + driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId))) + tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}")); + } + if (tagConfigWarnings.Count > 0) + { + if (_tagConfigValidationMode == TagConfigValidationMode.Error) + errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId))); + else + foreach (var x in tagConfigWarnings) + _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text); + } + + if (errors.Count > 0) + { + var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}")); + _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}", + errors.Count, summary); + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.Rejected, + DeploymentId: null, + RevisionHash: null, + Message: summary, + msg.CorrelationId), + Dispatch: null); + } + + // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough) + // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc + // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct + // sources are counted (the compile cache keys on source, so duplicates collapse to one unit). + // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard + // reject; this advisory rides in the Accepted Message so the UI can show it. + // advisory is declared outside the inner try so the Accepted reply below can still read it even + // when the guardrail throws. A failed estimate must NEVER block a valid deploy — the caller's + // failure branch is reserved for genuine seal/save failures. + string? advisory = null; + try + { + const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) + // Projected from the snapshot already in hand — Script.SourceCode used to be read three times. + var compiled = draft.Scripts + .Select(s => s.SourceCode) + .Where(src => !PassthroughScript.TryMatch(src, out _)) + .Distinct(StringComparer.Ordinal) + .Count(); + if (compiled > 0) + { + var estMiB = compiled * PerScriptMiB; + _log.Warning( + "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it", + compiled, estMiB); + advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)"; + } + } + catch (Exception ex) + { + // Guardrail is advisory-only — a failed estimate must never block a valid deploy. + _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped"); + advisory = null; + } + + // Composed from the validated snapshot, never from a second read: an edit that lands between + // validation and here is simply not in this artifact (R03:3.2). + var artifact = ConfigComposer.Compose(draft); + var deploymentId = DeploymentId.NewId(); + var revHash = RevisionHash.Parse(artifact.RevisionHash); + + db.Deployments.Add(new Deployment + { + DeploymentId = deploymentId.Value, + RevisionHash = artifact.RevisionHash, + Status = DeploymentStatus.Dispatching, + CreatedBy = msg.CreatedBy, + ArtifactBlob = artifact.Blob, + }); + + // Marker ConfigEdit row so the audit timeline shows the deployment snapshot. + db.ConfigEdits.Add(new ConfigEdit + { + EntityType = "Deployment", + EntityId = deploymentId.Value, + FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}", + EditedBy = msg.CreatedBy, + SourceNode = sourceNode, + }); + + await db.SaveChangesAsync(); + + // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so + // the operator sees the non-blocking strictness findings on a successful deploy. + var messageParts = new List(); + if (advisory is not null) messageParts.Add(advisory); + messageParts.AddRange(tagConfigWarnings.Select(x => x.Text)); + var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null; + + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.Accepted, + deploymentId, + revHash, + Message: acceptedMessage, + msg.CorrelationId), + new DispatchDeployment(deploymentId, revHash, msg.CorrelationId)); + } + + /// + /// Test Connect runs on the thread pool and pipes its reply straight to the requester: a probe + /// against a black-holed PLC (60 s clamp) must not suspend the singleton's mailbox (R03:3.1). Every + /// path inside the task yields a , so the requester never + /// sees a Status.Failure. + /// + private void HandleTestDriverConnect(TestDriverConnect msg) { var replyTo = Sender; if (!_probesByType.TryGetValue(msg.DriverType, out var probe)) @@ -396,6 +483,11 @@ public sealed class AdminOperationsActor : ReceiveActor return; } + Task.Run(() => ProbeAsync(probe, msg)).PipeTo(replyTo); + } + + private async Task ProbeAsync(IDriverProbe probe, TestDriverConnect msg) + { var clampedSec = Math.Clamp(msg.TimeoutSeconds, 1, 60); var timeout = TimeSpan.FromSeconds(clampedSec); using var cts = new CancellationTokenSource(timeout); @@ -405,28 +497,28 @@ public sealed class AdminOperationsActor : ReceiveActor { var result = await probe.ProbeAsync(msg.ConfigJson, timeout, cts.Token); sw.Stop(); - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( result.Ok, result.Message, result.Ok ? sw.Elapsed.TotalMilliseconds : (double?)null, - msg.CorrelationId)); + msg.CorrelationId); } catch (OperationCanceledException) { - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( false, $"Probe timed out after {clampedSec}s.", null, - msg.CorrelationId)); + msg.CorrelationId); } catch (Exception ex) { _log.Error(ex, "Probe for {DriverType} threw", msg.DriverType); - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( false, ex.Message, null, - msg.CorrelationId)); + msg.CorrelationId); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs new file mode 100644 index 00000000..95a2b0fc --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs @@ -0,0 +1,159 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:3.1 — the admin singleton's mailbox used to be suspended for the whole of every +/// ReceiveAsync: one Test Connect against a black-holed PLC (60 s clamp) or one large deploy +/// compose stalled every other admin operation fleet-wide, alarm acks included. Both now run off the +/// actor thread; only the "is one in flight" check stays serialised. +/// +public sealed class AdminOperationsActorMailboxTests : ControlPlaneActorTestBase +{ + private static readonly TimeSpan AckBudget = TimeSpan.FromSeconds(1); + + /// An alarm ack completes in under a second while a 60 s Test Connect is still pending. + [Fact] + public void Alarm_ack_completes_while_a_60s_test_connect_is_pending() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe("coord"); + var probe = new BlockingProbe(); + var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe })); + var connectClient = CreateTestProbe("connect-client"); + + connectClient.Send(actor, new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 60, Guid.NewGuid())); + probe.Started.Wait(PresenceBudget).ShouldBeTrue("the probe must be in flight before the ack is sent"); + + actor.Tell(new AcknowledgeAlarmCommand("alarm-1", "op", null, CorrelationId.NewId())); + var ack = ExpectMsg(AckBudget); + ack.Ok.ShouldBeTrue(); + + // Positive control: the probe reply still lands, with the right sender, once the probe returns. + connectClient.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + probe.Release(new DriverProbeResult(true, null, TimeSpan.FromMilliseconds(5))); + var result = connectClient.ExpectMsg(PresenceBudget); + result.Ok.ShouldBeTrue(); + result.LatencyMs.ShouldNotBeNull(); + } + + /// A probe that times out still answers its own sender with the clamped timeout message. + [Fact] + public void Test_connect_timeout_is_reported_to_its_sender() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe("coord"); + var probe = new BlockingProbe(); // never released: the CTS fires at the 1 s clamp floor + var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe })); + + actor.Tell(new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 1, Guid.NewGuid())); + + var result = ExpectMsg(PresenceBudget); + result.Ok.ShouldBeFalse(); + result.Message.ShouldNotBeNull(); + result.Message.ShouldContain("timed out"); + } + + /// + /// The deploy compose (snapshot + validate + gzip + row write) runs off the actor: an alarm ack + /// completes while the compose is parked on a gated DB context, and the deploy still completes + /// (Accepted + dispatched) once the gate opens. + /// + [Fact] + public void Alarm_ack_completes_while_a_deploy_compose_is_in_flight() + { + var gated = new GatedDbFactory(NewInMemoryDbFactory()); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty())); + var deployer = CreateTestProbe("deployer"); + + gated.Hold(); + deployer.Send(actor, new StartDeployment("joe", CorrelationId.NewId())); + gated.Entered.Wait(PresenceBudget).ShouldBeTrue("the compose must be parked before the ack is sent"); + + actor.Tell(new AcknowledgeAlarmCommand("alarm-2", "op", null, CorrelationId.NewId())); + ExpectMsg(AckBudget).Ok.ShouldBeTrue(); + + // Positive control: the deploy completes once released. + deployer.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + gated.Release(); + deployer.ExpectMsg(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.ExpectMsg(PresenceBudget); + } + + /// + /// Only the in-flight check is serialised — and it must still catch a second request that arrives + /// while the first compose has not yet written its row (the DB alone would let both through). + /// + [Fact] + public void Second_StartDeployment_during_an_in_flight_compose_is_refused() + { + var gated = new GatedDbFactory(NewInMemoryDbFactory()); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty())); + var first = CreateTestProbe("first"); + var second = CreateTestProbe("second"); + + gated.Hold(); + first.Send(actor, new StartDeployment("joe", CorrelationId.NewId())); + gated.Entered.Wait(PresenceBudget).ShouldBeTrue(); + + second.Send(actor, new StartDeployment("kim", CorrelationId.NewId())); + second.ExpectMsg(AckBudget).Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight); + + gated.Release(); + first.ExpectMsg(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.ExpectMsg(PresenceBudget); + } + + private sealed class BlockingProbe : IDriverProbe + { + private readonly TaskCompletionSource _tcs = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ManualResetEventSlim Started { get; } = new(); + public string DriverType => "Blocking"; + + public async Task ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct) + { + Started.Set(); + return await _tcs.Task.WaitAsync(ct); + } + + public void Release(DriverProbeResult result) => _tcs.TrySetResult(result); + } + + /// + /// Passes synchronous context creation through (the serialised in-flight check) and parks the + /// asynchronous one (the off-actor compose) on a gate while is in effect. + /// + private sealed class GatedDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private volatile TaskCompletionSource? _gate; + + public ManualResetEventSlim Entered { get; } = new(); + + public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _gate?.TrySetResult(); + + public OtOpcUaConfigDbContext CreateDbContext() => inner.CreateDbContext(); + + public async Task CreateDbContextAsync(CancellationToken cancellationToken = default) + { + Entered.Set(); + if (_gate is { } gate) await gate.Task.WaitAsync(cancellationToken); + return inner.CreateDbContext(); + } + } +} From 2e9160317f23847f01cbc29325b39480d64eb7b5 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:45:03 -0400 Subject: [PATCH 05/27] feat(scripting): deny System.Threading, Type reflection entry points and RuntimeHelpers in the sandbox (E2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R05:10 — the namespace deny-list covered System.Threading.Tasks but not System.Threading, so SemaphoreSlim / Monitor / SpinWait / Mutex / EventWaitHandle could park a script's pool thread past its budget (or touch a system-global kernel object); System.Type.GetType(string) / GetMethod* / InvokeMember and System.Runtime.CompilerServices.RuntimeHelpers resolved and reach a forbidden type without naming it syntactically. ForbiddenTypeAnalyzer now denies the whole System.Threading namespace with a type-granular allow-list (CancellationToken, Interlocked, Volatile — the sync-free primitives), adds a member deny-list for System.Type's reflection entry points (GetType/GetMethod(s)/InvokeMember/GetProperty(ies)/GetField(s)/ GetMember(s)/GetConstructor(s)/GetEvent(s)/GetDefaultMembers; instance object.GetType() stays allowed) and puts RuntimeHelpers on the type deny-list. Thread/ThreadPool/Timer leave the type list — the prefix rule reaches them. R05:11 — the allocation budget landed with E1's ScriptRunGuard (Scripting:MaxAllocBytesPerRun, default 64 MB): sampled around each run, breach quarantines. It is observation after the fact; a pre-emptive OOM guard still needs an out-of-process runner (follow-up issue in the lane report). The Monaco editor shares the analyzer, so DiagnoseTests pins every new form as an OTSCRIPT_FORBIDDEN marker and every allow-listed form as clean. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ForbiddenTypeAnalyzer.cs | 191 ++++++++++++------ .../ScriptSandboxHardeningTests.cs | 148 ++++++++++++++ .../ScriptAnalysis/DiagnoseTests.cs | 20 ++ 3 files changed, 300 insertions(+), 59 deletions(-) create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs index 005734b6..5f7d8140 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs @@ -17,13 +17,15 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// /// /// -/// Deny-list is the authoritative Phase 7 plan set: +/// Deny-list is the authoritative Phase 7 plan set, hardened since: /// System.IO, System.Net, System.Diagnostics, -/// System.Reflection, System.Threading.Thread, -/// System.Threading.Tasks (scripts are synchronous predicates — no -/// legitimate need to start background tasks; a Task.Run fan-out outlives -/// the evaluation timeout entirely), System.Runtime.InteropServices, -/// Microsoft.Win32. +/// System.Reflection, System.Threading.Tasks and the whole of +/// System.Threading (scripts are synchronous predicates — no legitimate need +/// to start background work, and no blocking primitive may park the evaluation's pool +/// thread past its budget; CancellationToken / Interlocked / +/// Volatile are allow-listed back in), System.Runtime.InteropServices, +/// System.Runtime.Loader, Microsoft.Win32; plus the member-granular +/// (System.Type's reflection entry points). /// /// /// Deny-list prefix match. System.Net catches System.Net.Http, @@ -63,13 +65,19 @@ public static class ForbiddenTypeAnalyzer "System.Net", "System.Diagnostics", // catches Process, ProcessStartInfo, EventLog, Trace/Debug file sinks "System.Reflection", - // System.Threading.Thread is NOT in this list: Thread's containing namespace is - // "System.Threading" (not "System.Threading.Thread"), so a prefix check on - // "System.Threading.Thread" never matches. Thread is denied type-granularly via - // ForbiddenFullTypeNames instead so the check actually fires. "System.Threading.Tasks", // Task.Run / Parallel — scripts are synchronous predicates // and have no legitimate need to start background work; // a Task fan-out outlives the evaluation timeout entirely. + // Listed BEFORE the parent prefix so its rejection names it. + "System.Threading", // Thread / ThreadPool / Timer (background work that outlives + // the timeout) AND every blocking primitive — SemaphoreSlim, + // Monitor, SpinWait, ManualResetEvent(Slim), Mutex, + // EventWaitHandle — which park the script's pool thread past + // its budget (the same orphan ScriptRunGuard exists to bound) + // and, for the named kernel objects, touch system-global + // state. The few sync-free primitives a script may + // legitimately name are allow-listed in + // AllowedTypesInForbiddenNamespaces. "System.Runtime.InteropServices", "System.Runtime.Loader", // AssemblyLoadContext + AssemblyDependencyResolver — // arbitrary DLL load into the host process. @@ -79,11 +87,55 @@ public static class ForbiddenTypeAnalyzer "Microsoft.Win32", // registry ]; + /// + /// Types that stay reachable although their namespace is in + /// . Matched by exact fully-qualified name against the + /// resolved type symbol (generic constructions unwrapped). The whole of System.Threading is + /// denied; these three are the sync-free primitives with no blocking surface — a + /// is read-only state, and + /// are single-instruction memory operations. + /// + public static readonly IReadOnlyList AllowedTypesInForbiddenNamespaces = + [ + "System.Threading.CancellationToken", + "System.Threading.Interlocked", + "System.Threading.Volatile", + ]; + + /// + /// Members scripts are NOT allowed to call on otherwise-allowed types, as + /// (declaring type, member name). System.Type must stay reachable + /// (typeof, Name, IsValueType are ordinary metadata reads) but its reflection + /// entry points obtain a type or member by NAME — Type.GetType("System.IO.File") reaches a + /// forbidden type without ever naming it syntactically, and GetMethod* / GetProperty + /// / InvokeMember / … hand back a System.Reflection handle (which pass 2 catches when + /// declared, but not when consumed inline). Instance object.GetType() is declared on + /// System.Object and is not matched. + /// + public static readonly IReadOnlyList<(string Type, string Member)> ForbiddenMembers = + [ + ("System.Type", "GetType"), + ("System.Type", "GetMethod"), + ("System.Type", "GetMethods"), + ("System.Type", "InvokeMember"), + ("System.Type", "GetProperty"), + ("System.Type", "GetProperties"), + ("System.Type", "GetField"), + ("System.Type", "GetFields"), + ("System.Type", "GetMember"), + ("System.Type", "GetMembers"), + ("System.Type", "GetConstructor"), + ("System.Type", "GetConstructors"), + ("System.Type", "GetEvent"), + ("System.Type", "GetEvents"), + ("System.Type", "GetDefaultMembers"), + ]; + /// /// Fully-qualified type names scripts are NOT allowed to reference, regardless of /// namespace. Used when a dangerous type shares its namespace with legitimate types - /// (e.g. System.Threading hosts both CancellationToken and - /// Thread) so a namespace-prefix rule would over-block. Matched by exact + /// (e.g. System.Runtime.CompilerServices hosts both MethodImplOptions and + /// RuntimeHelpers) so a namespace-prefix rule would over-block. Matched by exact /// fully-qualified name against the resolved type symbol — every member of /// the type (including read-only ones) is therefore rejected. /// @@ -99,19 +151,22 @@ public static class ForbiddenTypeAnalyzer /// System.Activator — CreateInstance is a /// reflection-equivalent escape that constructs a forbidden type by name /// without ever naming it syntactically. - /// System.Threading.Thread — raw thread creation bypasses the - /// per-evaluation timeout; denied type-granularly because its containing - /// namespace is System.Threading (shared with allowed types like - /// CancellationToken), so a namespace-prefix rule cannot reach it - /// without blocking unrelated types. - /// System.Threading.ThreadPool / System.Threading.Timer — - /// background-work vectors that outlive the per-evaluation timeout; same - /// threat as Task.Run. /// System.Runtime.CompilerServices.Unsafe — exposes raw type- /// reinterpretation (As<TFrom,TTo>, As<T>(object)) /// that bypasses the CLR type system without requiring an unsafe context at /// the call site; enables type-confusion and managed heap corruption. + /// System.Runtime.CompilerServices.RuntimeHelpers — + /// GetUninitializedObject constructs an instance of any type (a forbidden + /// one included) with no constructor and no syntactic type name beyond a + /// Type value; PrepareMethod / RunClassConstructor reach + /// runtime internals. /// + /// + /// System.Threading.Thread / ThreadPool / Timer used to be listed + /// here; since the whole System.Threading namespace is denied (with a type-granular + /// allow-list, see ) the prefix rule reaches + /// them first. + /// /// public static readonly IReadOnlyList ForbiddenFullTypeNames = [ @@ -119,26 +174,6 @@ public static class ForbiddenTypeAnalyzer "System.AppDomain", "System.GC", "System.Activator", - // System.Threading.Thread lives in the System.Threading namespace (shared with - // CancellationToken, SemaphoreSlim, etc.), so a namespace-prefix deny-list cannot - // target it without blocking those legitimate types. Denied type-granularly here. - "System.Threading.Thread", - // Broadening the references list to the BCL trusted-platform-assemblies set - // re-exposed two background-work vectors the original deny-list missed. Both live - // in System.Threading (shared with allowed sync primitives like CancellationToken / - // SemaphoreSlim), so they must be denied type-granularly: - // - // System.Threading.ThreadPool — QueueUserWorkItem / UnsafeQueueUserWorkItem - // re-introduce the background-fanout threat - // closed against System.Threading.Tasks. - // System.Threading.Timer — Timer(callback, …) schedules unbounded work - // that outlives the per-evaluation timeout. - // - // System.Runtime.Loader.AssemblyLoadContext is also covered, but at the namespace- - // prefix level above (System.Runtime.Loader) so future BCL additions to that - // namespace are denied by default. - "System.Threading.ThreadPool", - "System.Threading.Timer", // System.Runtime.CompilerServices.Unsafe exposes raw type-reinterpretation // (Unsafe.As, Unsafe.As(object)) that bypasses the CLR type system // without requiring an 'unsafe' compilation context at the call site. In a script, @@ -148,6 +183,11 @@ public static class ForbiddenTypeAnalyzer // compile-time-only types (CallerMemberNameAttribute, MethodImplAttribute, // MethodImplOptions) that scripts may legitimately reference. "System.Runtime.CompilerServices.Unsafe", + // RuntimeHelpers.GetUninitializedObject(Type) builds an instance of ANY type — a + // forbidden one obtained through Type.GetType (itself denied via ForbiddenMembers) + // included — without a constructor or a syntactic type name. Same namespace caveat + // as Unsafe above, so type-granular. + "System.Runtime.CompilerServices.RuntimeHelpers", ]; /// @@ -277,28 +317,61 @@ public static class ForbiddenTypeAnalyzer if (rejections.Any(r => r.Span == span && r.TypeName == typeName)) return; - var ns = typeSymbol.ContainingNamespace?.ToDisplayString() ?? string.Empty; - foreach (var forbidden in ForbiddenNamespacePrefixes) - { - if (ns == forbidden || ns.StartsWith(forbidden + ".", StringComparison.Ordinal)) - { - rejections.Add(new ForbiddenTypeRejection( - Span: span, - TypeName: typeName, - Namespace: ns, - Message: $"Type '{typeName}' is in the forbidden namespace '{ns}'. " + - $"Scripts cannot reach {forbidden}* per Phase 7 sandbox rules.")); - return; - } - } - - // Type-granular deny-list — dangerous types that live in the allow-listed - // System namespace and so cannot be caught by ForbiddenNamespacePrefixes. - // Matched on the full type name; OriginalDefinition unwraps any generic - // construction before naming. + // Full name of the (unwrapped) type — shared by the allow-list, the type deny-list and the + // member deny-list below. OriginalDefinition unwraps any generic construction before naming. var fullTypeName = typeSymbol.OriginalDefinition.ToDisplayString( SymbolDisplayFormat.FullyQualifiedFormat.WithGlobalNamespaceStyle( SymbolDisplayGlobalNamespaceStyle.Omitted)); + + var ns = typeSymbol.ContainingNamespace?.ToDisplayString() ?? string.Empty; + + // Member deny-list first: the containing type (System.Type) is allowed, so neither list + // below would ever look at the member name. + if (symbol is IMethodSymbol method) + { + var declaring = method.ContainingType.OriginalDefinition.ToDisplayString( + SymbolDisplayFormat.FullyQualifiedFormat.WithGlobalNamespaceStyle( + SymbolDisplayGlobalNamespaceStyle.Omitted)); + foreach (var (type, member) in ForbiddenMembers) + { + if (declaring == type && method.Name == member) + { + rejections.Add(new ForbiddenTypeRejection( + Span: span, + TypeName: $"{type}.{member}", + Namespace: ns, + Message: $"Member '{type}.{member}' is on the sandbox forbidden-member deny-list: " + + "it obtains a type or member by name, which bypasses the type allow-list " + + "(see ForbiddenTypeAnalyzer.ForbiddenMembers for the authoritative list).")); + return; + } + } + } + + // A type on the allow-list stays reachable even though its namespace is denied; it is still + // subject to the type-granular deny-list below (nothing is on both today). + var allowListed = AllowedTypesInForbiddenNamespaces.Contains(fullTypeName, StringComparer.Ordinal); + + if (!allowListed) + { + foreach (var forbidden in ForbiddenNamespacePrefixes) + { + if (ns == forbidden || ns.StartsWith(forbidden + ".", StringComparison.Ordinal)) + { + rejections.Add(new ForbiddenTypeRejection( + Span: span, + TypeName: typeName, + Namespace: ns, + Message: $"Type '{typeName}' is in the forbidden namespace '{ns}'. " + + $"Scripts cannot reach {forbidden}* per Phase 7 sandbox rules.")); + return; + } + } + } + + // Type-granular deny-list — dangerous types that live in an allow-listed namespace + // (System, System.Runtime.CompilerServices) and so cannot be caught by + // ForbiddenNamespacePrefixes. foreach (var forbiddenType in ForbiddenFullTypeNames) { if (fullTypeName == forbiddenType) diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs new file mode 100644 index 00000000..5c932832 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs @@ -0,0 +1,148 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; + +/// +/// Sandbox hardening (review R05:10): System.Threading is denied as a namespace with a +/// type-granular allow-list (CancellationToken, Interlocked, Volatile) so a +/// script can no longer park its pool thread on a blocking primitive or touch a system-global kernel +/// object; System.Type's reflection entry points (GetType(string), GetMethod*, +/// InvokeMember, …) and System.Runtime.CompilerServices.RuntimeHelpers are denied so a +/// forbidden type cannot be reached by name without ever being named syntactically. +/// +[Trait("Category", "Unit")] +public sealed class ScriptSandboxHardeningTests +{ + private static ScriptSandboxViolationException Rejects(string source) => + Should.Throw(() => + ScriptEvaluator.Compile(source)); + + // ── System.Threading blocking / synchronisation primitives ─────────────────────────────── + + [Fact] + public void Rejects_SemaphoreSlim_Wait() + { + var ex = Rejects("new System.Threading.SemaphoreSlim(0).Wait(); return 0;"); + ex.Rejections.ShouldContain(r => r.TypeName.Contains("SemaphoreSlim") && r.Namespace == "System.Threading"); + } + + [Fact] + public void Rejects_SpinWait_SpinUntil() + => Rejects("System.Threading.SpinWait.SpinUntil(() => false); return 0;"); + + [Fact] + public void Rejects_Monitor() + => Rejects("var o = new object(); System.Threading.Monitor.Enter(o); return 0;"); + + [Fact] + public void Rejects_named_Mutex_system_global_kernel_object() + => Rejects("using var m = new System.Threading.Mutex(false, \"Global\\\\otopcua\"); return 0;"); + + [Fact] + public void Rejects_ManualResetEventSlim() + => Rejects("var e = new System.Threading.ManualResetEventSlim(false); e.Wait(); return 0;"); + + [Fact] + public void Rejects_EventWaitHandle() + => Rejects("var e = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset); return 0;"); + + [Fact] + public void Rejects_Thread_ThreadPool_and_Timer_still() + { + Rejects("var t = new System.Threading.Thread(() => { }); return 0;"); + Rejects("System.Threading.ThreadPool.QueueUserWorkItem(_ => { }); return 0;"); + Rejects("var t = new System.Threading.Timer(_ => { }); return 0;"); + } + + [Fact] + public void Rejects_Thread_Sleep() + => Rejects("System.Threading.Thread.Sleep(10); return 0;"); + + // ── the allow-list inside System.Threading ─────────────────────────────────────────────── + + [Fact] + public async Task Allows_CancellationToken() + { + using var ev = ScriptEvaluator.Compile( + "var t = System.Threading.CancellationToken.None; return t.IsCancellationRequested ? 1 : 0;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(0); + } + + [Fact] + public async Task Allows_Interlocked() + { + using var ev = ScriptEvaluator.Compile( + "var x = 41; System.Threading.Interlocked.Increment(ref x); return x;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(42); + } + + [Fact] + public async Task Allows_Volatile() + { + using var ev = ScriptEvaluator.Compile( + "var x = 42; return System.Threading.Volatile.Read(ref x);"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(42); + } + + // ── System.Type reflection entry points ────────────────────────────────────────────────── + + [Fact] + public void Rejects_Type_GetType_by_name() + { + var ex = Rejects("var t = System.Type.GetType(\"System.IO.File\"); return t is null ? 0 : 1;"); + ex.Rejections.ShouldContain(r => r.Message.Contains("GetType")); + } + + [Fact] + public void Rejects_Type_GetMethod() + => Rejects("var m = typeof(string).GetMethod(\"Trim\"); return m is null ? 0 : 1;"); + + [Fact] + public void Rejects_Type_GetMethods() + => Rejects("return typeof(string).GetMethods().Length;"); + + [Fact] + public void Rejects_Type_InvokeMember() + => Rejects("""return (int)typeof(string).InvokeMember("Length", System.Reflection.BindingFlags.GetProperty, null, "abc", null)!;"""); + + [Fact] + public void Rejects_Type_GetProperty_GetField_GetMember_GetConstructor_GetEvent() + { + Rejects("return typeof(string).GetProperty(\"Length\") is null ? 0 : 1;"); + Rejects("return typeof(string).GetField(\"Empty\") is null ? 0 : 1;"); + Rejects("return typeof(string).GetMember(\"Length\").Length;"); + Rejects("return typeof(string).GetConstructors().Length;"); + Rejects("return typeof(string).GetEvent(\"x\") is null ? 0 : 1;"); + } + + [Fact] + public async Task Allows_instance_GetType_and_Type_metadata_reads() + { + // object.GetType() is declared on System.Object, and Type.Name / IsValueType are plain metadata + // reads — neither is a reflection entry point, and both are common in authored scripts. + using var ev = ScriptEvaluator.Compile( + "object o = 42; var t = o.GetType(); return t.IsValueType && typeof(int).Name == \"Int32\" ? 1 : 0;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(1); + } + + // ── RuntimeHelpers ─────────────────────────────────────────────────────────────────────── + + [Fact] + public void Rejects_RuntimeHelpers() + { + var ex = Rejects( + "var o = System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(string)); return o is null ? 0 : 1;"); + ex.Rejections.ShouldContain(r => r.TypeName.Contains("RuntimeHelpers")); + } + + [Fact] + public async Task Allows_compile_time_only_CompilerServices_attributes() + { + // The namespace stays open for the harmless compile-time types (MethodImplOptions etc.). + using var ev = ScriptEvaluator.Compile( + "var o = System.Runtime.CompilerServices.MethodImplOptions.NoInlining; return (int)o;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(8); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs index 21460637..94784a45 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs @@ -31,6 +31,26 @@ public sealed class DiagnoseTests => Diag("""var c = new System.Net.Http.HttpClient(); return 0;""") .ShouldContain(x => x.Severity == 8); + // Sandbox hardening parity (R05:10): the editor runs the SAME ForbiddenTypeAnalyzer the publish gate + // runs, so every newly-denied form must surface as an OTSCRIPT_FORBIDDEN marker here — editor accepts + // ⇔ publish accepts. One case per deny mechanism (namespace, member, type), plus the allow-list. + [Theory] + [InlineData("""new System.Threading.SemaphoreSlim(0).Wait(); return 0;""")] // System.Threading namespace + [InlineData("""System.Threading.SpinWait.SpinUntil(() => false); return 0;""")] + [InlineData("""var t = System.Type.GetType("System.IO.File"); return t is null ? 0 : 1;""")] // forbidden member + [InlineData("""return typeof(string).GetMethods().Length;""")] + [InlineData("""var o = System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(string)); return 0;""")] // forbidden type + public void Hardened_sandbox_forms_are_flagged_in_the_editor(string code) + => Diag(code).ShouldContain(x => x.Code == "OTSCRIPT_FORBIDDEN" && x.Severity == 8); + + [Theory] + [InlineData("""var t = System.Threading.CancellationToken.None; return t.IsCancellationRequested ? 1 : 0;""")] + [InlineData("""var x = 41; System.Threading.Interlocked.Increment(ref x); return x;""")] + [InlineData("""var x = 42; return System.Threading.Volatile.Read(ref x);""")] + [InlineData("""object o = 42; return o.GetType().IsValueType ? 1 : 0;""")] + public void Allow_listed_forms_are_not_flagged_in_the_editor(string code) + => Diag(code).ShouldBeEmpty(); + [Fact] public void Dynamic_tag_path_is_flagged() => Diag("""var p = "A"; return ctx.GetTag(p).Value;""") .ShouldContain(x => x.Message.Contains("literal")); From f5579e6ef2ba48358814ea93f143d276d9a72867 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:15:05 -0400 Subject: [PATCH 06/27] fix(host): make a failed OPC UA SDK start visible and stop the SDK on StopAsync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed SDK start (port in use, PKI unwritable, bad OpcUa:* override) was one Error line: the node stayed /health/ready 200, joined its mesh, could be elected driver Primary and acked deploys through the Null sink while every client got connection-refused. - IOpcUaEndpointState / OpcUaEndpointState: the start outcome is a fact the hosted service maintains (NotStarted / Started / Failed / Stopped + reason). - OpcUaEndpointHealthCheck ("opcua-endpoint", Ready + Active) is Unhealthy unless Started; registered by AddOtOpcUaHealth(hasAdmin, hasDriver) on driver-role nodes. - A driver-only node rethrows the start failure (Program.cs's fatal handler exits non-zero so the supervisor restarts it); a fused node keeps its AdminUI up but is visibly not ready. - StopAsync detaches the sinks and then STOPS the SDK (the port closes, clients get connection-refused rather than a stale, inert endpoint); DisposeAsync is idempotent with it. - The ServiceLevel calculator / redundancy election live in Cluster + Runtime (not this lane); the fact is exposed for the Wave 3 follow-up to consume. Tests: port-in-use ⇒ state Failed, probe Unhealthy, real /health/ready and /health/active 503 (positive control: 200 once Started); driver-only rethrows; StopAsync closes the port and DisposeAsync twice is a no-op; the probe registration is on both tiers for driver nodes only. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Health/HealthEndpoints.cs | 21 +- .../Health/OpcUaEndpointHealthCheck.cs | 45 ++++ .../OpcUa/IOpcUaEndpointState.cs | 73 ++++++ .../OpcUa/OtOpcUaServerHostedService.cs | 124 ++++++++-- src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 10 +- .../Health/ActiveTierRegistrationTests.cs | 28 +++ ...cUaServerHostedServiceStartFailureTests.cs | 213 ++++++++++++++++++ 7 files changed, 492 insertions(+), 22 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs index f7a740bd..2f57158f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs @@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Host.Configuration; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; namespace ZB.MOM.WW.OtOpcUa.Host.Health; @@ -28,11 +29,29 @@ public static class HealthEndpoints /// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to /// true so admin/fused callers (including the test harnesses) keep the probe unchanged. /// + /// + /// Whether this node carries the driver role and therefore serves an OPC UA endpoint. When + /// true the joins the ready + active tiers (R03:2.1), reading + /// the the host registers alongside OtOpcUaServerHostedService. + /// Defaults to false so admin-only callers and the existing harnesses register nothing new. + /// /// The same service collection, for chaining. - public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true) + public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false) { var checks = services.AddHealthChecks(); + // A driver node's whole purpose is its OPC UA endpoint, so a failed SDK start must take it out of + // the ready AND active tiers: readiness because it cannot serve, active because a load balancer or + // the fleet must not route to it. Factory-registered so the state singleton is resolved per probe. + if (hasDriver) + { + checks.Add(new HealthCheckRegistration( + OpcUaEndpointHealthCheck.Name, + sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService()), + failureStatus: null, + tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })); + } + // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config // from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so // the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs new file mode 100644 index 00000000..8caa69f5 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Health; + +/// +/// Ready + Active tier probe for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is +/// listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy +/// one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets +/// connection-refused. +/// +/// +/// Registered on driver-role nodes only (); an +/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. +/// +public sealed class OpcUaEndpointHealthCheck : IHealthCheck +{ + /// The registration name, as it appears in the health JSON. + public const string Name = "opcua-endpoint"; + + private readonly IOpcUaEndpointState _state; + + /// Initializes a new instance of the class. + /// The endpoint lifecycle fact the hosted service maintains. + public OpcUaEndpointHealthCheck(IOpcUaEndpointState state) + { + ArgumentNullException.ThrowIfNull(state); + _state = state; + } + + /// + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + var data = new Dictionary { ["status"] = _state.Status.ToString() }; + var result = _state.Status switch + { + OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data), + OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data), + OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data), + _ => HealthCheckResult.Unhealthy( + "OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data), + }; + return Task.FromResult(result); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs new file mode 100644 index 00000000..c2377ea1 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs @@ -0,0 +1,73 @@ +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// Lifecycle of this node's OPC UA endpoint as drives it. +public enum OpcUaEndpointStatus +{ + /// The hosted service has not run StartAsync yet. + NotStarted, + + /// The SDK is listening and the address-space / ServiceLevel sinks are bound. + Started, + + /// The SDK failed to start (port in use, PKI unwritable, bad OpcUa:* override, …). + Failed, + + /// The hosted service stopped the SDK; the port is released. + Stopped, +} + +/// +/// The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed +/// start was one Error line: /health/ready stayed 200, the node joined its mesh, could be +/// elected driver Primary and acked deploys as Applied through the Null sink — while clients got +/// connection-refused from the node the fleet called healthy. +/// +/// +/// Consumed by OpcUaEndpointHealthCheck (Ready + Active tiers). The redundancy election and the +/// ServiceLevel calculator do not read it yet — they live outside the Host (Cluster / +/// Runtime); feeding them is the Wave 3 follow-up the lane report names. Until then a fused +/// node whose SDK failed is visible (503) but still Primary-eligible. +/// +public interface IOpcUaEndpointState +{ + /// The endpoint's current lifecycle status. + OpcUaEndpointStatus Status { get; } + + /// Why the start failed, when is . + string? FailureReason { get; } +} + +/// Mutable ; written by the hosted service, read by the health check. +public sealed class OpcUaEndpointState : IOpcUaEndpointState +{ + private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted; + private volatile string? _failureReason; + + /// + public OpcUaEndpointStatus Status => _status; + + /// + public string? FailureReason => _failureReason; + + /// Records a successful SDK start. + public void MarkStarted() + { + _failureReason = null; + _status = OpcUaEndpointStatus.Started; + } + + /// Records a failed SDK start. + /// The failure, as the health probe should report it. + public void MarkFailed(string reason) + { + _failureReason = reason; + _status = OpcUaEndpointStatus.Failed; + } + + /// Records that the hosted service stopped the SDK. + public void MarkStopped() + { + _failureReason = null; + _status = OpcUaEndpointStatus.Stopped; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs index 79f62cd8..609b0087 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Authorization; @@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl private readonly IDriverPrimaryRoleSource? _primaryRoleSource; private readonly HistorianUpdaterHolder _historianUpdater; private readonly IHistoryUpdateRoleView? _historyUpdateRoleView; + private readonly OpcUaEndpointState _endpointState; + private readonly bool _rethrowStartFailure; private OpcUaApplicationHost? _appHost; private OtOpcUaSdkServer? _server; @@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl /// The HistoryUpdate verdict DriverHostActor publishes (the /// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every /// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow). + /// The endpoint lifecycle fact this service maintains for the + /// opcua-endpoint health probe (R03:2.1). Optional: a private instance is kept when none is registered, + /// so a harness that never probes health is unchanged. + /// The node's resolved roles. A node with the driver role and NO admin role exists only + /// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and + /// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through + /// the health probe instead. Null (harnesses) behaves as fused. public OtOpcUaServerHostedService( IOptions options, DeferredAddressSpaceSink deferredSink, @@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl ILoggerFactory loggerFactory, IDriverPrimaryRoleSource? primaryRoleSource = null, HistorianUpdaterHolder? historianUpdater = null, - IHistoryUpdateRoleView? historyUpdateRoleView = null) + IHistoryUpdateRoleView? historyUpdateRoleView = null, + OpcUaEndpointState? endpointState = null, + EffectiveRoles? roles = null) { ArgumentNullException.ThrowIfNull(aclOptions); ArgumentNullException.ThrowIfNull(time); @@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl _primaryRoleSource = primaryRoleSource; _historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None; _historyUpdateRoleView = historyUpdateRoleView; + _endpointState = endpointState ?? new OpcUaEndpointState(); + _rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false }; _logger = loggerFactory.CreateLogger(); } @@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl } catch (Exception ex) { + // R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy + // (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits + // so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct + // the OpcUa:* config without a process bounce — but it is visibly not ready. + _endpointState.MarkFailed(ex.Message); + await ReleaseFailedStartAsync().ConfigureAwait(false); + if (_rethrowStartFailure) + { + _logger.LogCritical(ex, + "OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " + + "node justifies staying up; failing host start so the supervisor restarts the process"); + throw; + } _logger.LogError(ex, - "OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op"); - // Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot. - // Operators see the failure via the logs + can correct config without a process bounce - // of the whole binary. + "OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " + + "OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section"); return; } @@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl { _logger.LogWarning( "OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null"); + _endpointState.MarkFailed("SDK reported started but the node manager is null"); return; } @@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl { _logger.LogError(ex, "OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues"); + _endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message); try { - await StopAsync(cancellationToken).ConfigureAwait(false); + DetachSinks(); } catch (Exception cleanupError) { _logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure"); } - var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync( - async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false), - () => _appHost.DisposeAsync(), _logger).ConfigureAwait(false); - if (released) - { - _appHost = null; - _server = null; - } + await ReleaseEndpointAsync().ConfigureAwait(false); return; } @@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // attached, so the first tick cannot land on a half-wired server. StartRevalidationTimer(); + _endpointState.MarkStarted(); _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound"); } + /// + /// Releases an SDK whose StartAsync threw. The application instance may hold a half-opened + /// listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the + /// transport is up); disposing it is the only way to be sure. The state stays Failed. + /// + private async Task ReleaseFailedStartAsync() + { + var appHost = _appHost; + _appHost = null; + _server = null; + if (appHost is null) return; + try + { + await appHost.DisposeAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw"); + } + } + + /// + /// Stops the SDK and disposes the application host, releasing endpoint ownership only when both + /// succeeded (): on a failure the + /// fields are retained so retries at host disposal. + /// + private async Task ReleaseEndpointAsync() + { + var server = _server; + var appHost = _appHost; + if (server is null && appHost is null) return; + var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync( + async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); }, + () => appHost?.DisposeAsync() ?? ValueTask.CompletedTask, + _logger).ConfigureAwait(false); + if (released) + { + _appHost = null; + _server = null; + } + } + /// /// The ServerHistorian:HonorReturnBounds kill switch StartAsync applies to the node manager /// (default true when the section or key is absent). Exposed so the wiring is testable without booting the SDK. @@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl } /// - /// Stops the OPC UA server asynchronously. + /// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes. /// + /// + /// R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown, + /// the other hosted services' StopAsync), serving the last-written values with an emptied authority: + /// reads succeeded on stale data while writes got "unavailable", and clients saw a server that was + /// up but inert instead of the connection-refused that triggers their failover. + /// stays idempotent with this: ownership is nulled once the stop succeeded. + /// /// Cancellation token. /// A task that represents the asynchronous operation. - public Task StopAsync(CancellationToken cancellationToken) + public async Task StopAsync(CancellationToken cancellationToken) + { + DetachSinks(); + await ReleaseEndpointAsync().ConfigureAwait(false); + if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped(); + } + + /// + /// Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor + /// or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running + /// until . + /// + private void DetachSinks() { // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose // authority and gate have already been cleared is the shape this whole teardown block exists to @@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // request rather than let the seams fall back to their role checks — a stopping node should shed // enforcement in the same order it acquired it. _server?.SetDataPlaneAuthorizer(null); - return Task.CompletedTask; } /// @@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl public async ValueTask DisposeAsync() { // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must - // not leave a tick armed against a server that is about to go away. + // not leave a tick armed against a server that is about to go away. A StopAsync that released the + // endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left + // them set, and this is the retry. _revalidationTimer?.Dispose(); _revalidationTimer = null; - if (_appHost is not null) await _appHost.DisposeAsync(); + var appHost = _appHost; + _appHost = null; + _server = null; + if (appHost is not null) await appHost.DisposeAsync(); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 6404504e..d326d1b1 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -452,6 +452,11 @@ if (hasDriver) // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once. builder.Services.TryAddSingleton>(sp => () => sp.GetRequiredService()); + // The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint + // health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the + // interface, so the writer and the probe cannot disagree. + builder.Services.AddSingleton(); + builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); } @@ -554,8 +559,9 @@ if (hasAdmin) builder.Services.AddOtOpcUaSecrets(builder.Configuration); // hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so -// probing it would throw when the readiness endpoint resolves the factory. -builder.Services.AddOtOpcUaHealth(hasAdmin); +// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the +// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green. +builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver); builder.Services.AddOtOpcUaObservability(builder.Configuration); // gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs index 1a9730ce..d880e7b3 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs @@ -49,6 +49,34 @@ public sealed class ActiveTierRegistrationTests ClusterPrimaryRegistration(hasAdmin).Tags.ShouldContain(ZbHealthTags.Active); } + /// + /// R03:2.1 — the OPC UA endpoint probe is on BOTH tiers of a driver node (it cannot serve, and must not + /// be routed to) and absent from an admin-only node, which starts no endpoint to be degraded by. + /// + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Opcua_endpoint_probe_is_registered_on_ready_and_active_for_driver_nodes_only(bool hasDriver) + { + var registrations = new ServiceCollection() + .AddOtOpcUaHealth(hasAdmin: true, hasDriver: hasDriver) + .BuildServiceProvider() + .GetRequiredService>() + .Value.Registrations + .Where(r => r.Name == OpcUaEndpointHealthCheck.Name) + .ToList(); + + if (!hasDriver) + { + registrations.ShouldBeEmpty(); + return; + } + + var registration = registrations.ShouldHaveSingleItem(); + registration.Tags.ShouldContain(ZbHealthTags.Ready); + registration.Tags.ShouldContain(ZbHealthTags.Active); + } + [Fact] public void Active_tier_check_is_not_on_the_ready_tier() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs new file mode 100644 index 00000000..341595d7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs @@ -0,0 +1,213 @@ +using System.Net; +using System.Net.Sockets; +using Akka.Hosting; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.Health; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// R03:2.1 / 1.4 — a failed OPC UA SDK start used to be one Error line: the node stayed Ready, joined +/// its mesh, and acked deploys through the Null sink while clients got connection-refused. The start +/// outcome is now a fact () the ready/active tiers read, a driver-only +/// node rethrows so the process exits, and StopAsync actually stops the SDK. +/// +public sealed class OtOpcUaServerHostedServiceStartFailureTests +{ + private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30); + + /// + /// Port already bound ⇒ the SDK cannot listen ⇒ the state is Failed, the probe is Unhealthy and the + /// real /health/ready endpoint answers 503 — while StartAsync itself returns (fused node). + /// Positive control: the same endpoint answers 200 once the state is Started. + /// + [Fact] + public async Task Port_in_use_marks_the_endpoint_failed_and_health_ready_answers_503() + { + using var deadline = new CancellationTokenSource(Deadline); + using var squatter = SquatAllInterfaces(out var port); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var fused = EffectiveRoles.Resolve(new[] { "admin", "driver" }, envRoles: null); + + var service = NewService(port, pki, state, fused); + try + { + await service.StartAsync(deadline.Token); // must NOT throw on a fused node + + state.Status.ShouldBe(OpcUaEndpointStatus.Failed); + state.FailureReason.ShouldNotBeNullOrWhiteSpace(); + + var probe = await new OpcUaEndpointHealthCheck(state).CheckHealthAsync(new HealthCheckContext()); + probe.Status.ShouldBe(HealthStatus.Unhealthy); + probe.Description.ShouldNotBeNull(); + probe.Description.ShouldContain("failed to start"); + + await using var web = BuildHealthHost(state); + await web.StartAsync(deadline.Token); + using var http = new HttpClient { BaseAddress = new Uri(web.Urls.First()) }; + + (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + (await http.GetAsync("/health/active", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + + // Positive control on the endpoint: the same host answers 200 for a started endpoint, so the + // 503 above measured the probe and not a broken health pipeline. + state.MarkStarted(); + (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.OK); + await web.StopAsync(deadline.Token); + } + finally + { + await service.DisposeAsync(); + squatter.Stop(); + Cleanup(pki); + } + } + + /// A driver-only node has nothing else to stay up for: the failed start propagates. + [Fact] + public async Task Driver_only_node_rethrows_when_the_sdk_fails_to_start() + { + using var deadline = new CancellationTokenSource(Deadline); + using var squatter = SquatAllInterfaces(out var port); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var driverOnly = EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null); + + var service = NewService(port, pki, state, driverOnly); + try + { + await Should.ThrowAsync(() => service.StartAsync(deadline.Token)); + state.Status.ShouldBe(OpcUaEndpointStatus.Failed); + } + finally + { + await service.DisposeAsync(); + squatter.Stop(); + Cleanup(pki); + } + } + + /// + /// A clean start is Started and listening; StopAsync closes the port (connection refused, not + /// a stale-but-listening endpoint) and marks Stopped; DisposeAsync twice afterwards is a no-op. + /// + [Fact] + public async Task StopAsync_closes_the_port_and_DisposeAsync_is_idempotent() + { + using var deadline = new CancellationTokenSource(Deadline); + var port = FreePort(); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var service = NewService(port, pki, state, EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null)); + try + { + await service.StartAsync(deadline.Token); + state.Status.ShouldBe(OpcUaEndpointStatus.Started); + using (var client = new TcpClient()) + { + await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token); + client.Connected.ShouldBeTrue("positive control: the endpoint is listening after StartAsync"); + } + + await service.StopAsync(deadline.Token); + + state.Status.ShouldBe(OpcUaEndpointStatus.Stopped); + using (var client = new TcpClient()) + { + await Should.ThrowAsync(async () => await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token)); + } + + await service.DisposeAsync(); + await service.DisposeAsync(); // idempotent + } + finally + { + await service.DisposeAsync(); + Cleanup(pki); + } + } + + private static OtOpcUaServerHostedService NewService(int port, string pki, OpcUaEndpointState state, EffectiveRoles roles) => + new( + Options.Create(new OpcUaApplicationHostOptions + { + ApplicationName = "OtOpcUa.StartFailure", + ApplicationUri = $"urn:OtOpcUa.StartFailure:{Guid.NewGuid():N}", + OpcUaPort = port, PublicHostname = "localhost", PkiStoreRoot = pki, + }), + new DeferredAddressSpaceSink(), new DeferredServiceLevelPublisher(), + NullOpcUaUserAuthenticator.Instance, NullHistorianDataSource.Instance, + EmptyDataPlaneAuthoritySource.Instance, new ConfigurationBuilder().Build(), + () => throw new NotSupportedException("no actor system required"), new ActorRegistry(), + null, Options.Create(new AclOptions()), TimeProvider.System, NullLoggerFactory.Instance, + endpointState: state, roles: roles); + + /// The production health registration (driver role) over the supplied state, on a loopback Kestrel. + private static WebApplication BuildHealthHost(OpcUaEndpointState state) + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + builder.Logging.ClearProviders(); + builder.Services.AddSingleton(state); + builder.Services.AddOtOpcUaHealth(hasAdmin: false, hasDriver: true); + // The akka + cluster-primary probes need a running ActorSystem; keep only the probe under test so + // the status measured is the endpoint's. (The registration shape itself is pinned separately.) + builder.Services.Configure(o => + { + foreach (var r in o.Registrations.Where(r => r.Name != OpcUaEndpointHealthCheck.Name).ToList()) + o.Registrations.Remove(r); + }); + var app = builder.Build(); + app.MapOtOpcUaHealth(); + return app; + } + + /// + /// Holds a port on the IPv4 AND IPv6 wildcard addresses. The SDK binds its listener on the + /// wildcards, and on macOS a loopback-only squatter does not conflict with a wildcard bind (both + /// sockets carry SO_REUSEADDR), so a loopback squatter would let the SDK start — the first run of + /// this test proved exactly that. The SDK also only fails when NEITHER family binds. + /// + private static TcpListener SquatAllInterfaces(out int port) + { + var squatter = new TcpListener(IPAddress.IPv6Any, 0); + squatter.Server.DualMode = true; + squatter.ExclusiveAddressUse = true; + squatter.Start(); + port = ((IPEndPoint)squatter.LocalEndpoint).Port; + return squatter; + } + + private static int FreePort() + { + using var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + var port = ((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); + return port; + } + + private static string NewPki() => Path.Combine(Path.GetTempPath(), $"otopcua-startfail-{Guid.NewGuid():N}"); + + private static void Cleanup(string pki) + { + if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true); + } +} From 6cbf763719ba5eebb339947512a4ecafd395ac8c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:17:30 -0400 Subject: [PATCH 07/27] fix(alarms): engine cache prune + bounded dispose, source Warning, cached dead-letter depth, script-log rate limit (E3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R05:1/12 ScriptedAlarmEngine: _valueCache is cleared in LoadAsync (it is re-seeded for the referenced paths) so retired tags no longer accrete across deploys; OnUpstreamChange no longer writes the cache after Dispose. - R05:3 ScriptedAlarmEngine.Dispose drains in-flight work for at most 5 s (DisposeDrainTimeout), then cancels it through a shutdown token every DrainDirtyAsync/ShelvingCheckAsync now runs on, logs a Warning and gives one second of grace — a stalled IAlarmStateStore.SaveAsync can no longer hang host shutdown. A save that completes inside the bound is still drained (positive control test). - R05:6 ScriptedAlarmSource logs a throwing OnAlarmEvent subscriber at Warning (naming the alarm) instead of swallowing it. - R05:7 LocalDbStoreAndForwardSink keeps a cached dead-letter counter in step with every dead-letter/revive/purge path and resyncs it with the queued counter on the same cadence; GetStatus no longer runs a COUNT(*) scan. - R05:9 ScriptLogRateLimiter (20 per source per minute, fixed window, one summary entry naming the dropped count when the window rolls) behind ScriptLogTopicSink (key = script|vtag|alarm) and ScriptLogCompanionSink (key = script name). Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../LocalDbStoreAndForwardSink.cs | 59 +++++-- .../ScriptedAlarmEngine.cs | 53 +++++- .../ScriptedAlarmSource.cs | 19 ++- .../ScriptLogCompanionSink.cs | 23 ++- .../ScriptLogRateLimiter.cs | 123 ++++++++++++++ .../ScriptLogTopicSink.cs | 49 +++++- .../LocalDbStoreAndForwardSinkTests.cs | 34 ++++ .../ScriptedAlarmEngineLifecycleTests.cs | 159 ++++++++++++++++++ .../ScriptedAlarmSourceLoggingTests.cs | 59 +++++++ .../ScriptLogRateLimiterTests.cs | 137 +++++++++++++++ 10 files changed, 683 insertions(+), 32 deletions(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs index 8a9b9ec4..6d239f75 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs @@ -92,6 +92,12 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl private bool? _lastGateDecision; private long _queuedRowCount; + // Cached dead-letter depth (R05:7). GetStatus used to run a synchronous COUNT(*) table scan per + // call — one per /healthz probe and per AdminUI poll on the shared LocalDb file. Kept in step by + // every path that flips dead_lettered (drain outcomes, corrupt rows, RetryDeadLettered, retention + // purge) and resynced from storage on the same cadence as _queuedRowCount, so a row the pair peer + // replicates in is visible after the next resync rather than never. + private long _deadLetterRowCount; // Probe counter — incremented every time we actually issue a real COUNT(*) for // capacity enforcement. Public for test instrumentation only. private long _capacityProbeCount; @@ -145,9 +151,16 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl _clock = clock ?? (() => DateTime.UtcNow); _drainGate = drainGate ?? (static () => true); - _queuedRowCount = ProbeQueuedRowCount(); + _queuedRowCount = ProbeRowCount(deadLettered: false); + _deadLetterRowCount = ProbeRowCount(deadLettered: true); } + /// Test-only: re-sync both cached counters (queued + dead-lettered) from storage — the same + /// resync the enqueue cadence drives every enqueues. + /// Cancellation token. + /// A task that completes once both counters match storage. + public Task DebugResyncCountersAsync(CancellationToken ct) => ResyncRowCountsAsync(ct); + /// /// Start the background drain worker. Not started automatically so tests can /// drive deterministically. @@ -260,7 +273,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl // has a second way to fall behind that no local code path can observe. if (enqueuesSinceResync >= ResyncEnqueueInterval) { - await ResyncQueuedRowCountAsync(ct).ConfigureAwait(false); + await ResyncRowCountsAsync(ct).ConfigureAwait(false); cached = Interlocked.Read(ref _queuedRowCount); Interlocked.Exchange(ref _enqueuesSinceResync, 0); } @@ -273,22 +286,26 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl await EnforceCapacityAsync(ct).ConfigureAwait(false); } - /// Synchronously query COUNT(*) of non-dead-lettered rows. Used at startup. - private long ProbeQueuedRowCount() + /// Synchronously query COUNT(*) of queued or dead-lettered rows. Used at startup. + private long ProbeRowCount(bool deadLettered) { Interlocked.Increment(ref _capacityProbeCount); using var conn = _db.CreateConnection(); using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0"; + cmd.CommandText = deadLettered + ? "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1" + : "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0"; return (long)(cmd.ExecuteScalar() ?? 0L); } - /// Re-sync the in-memory counter from storage (async path). - private async Task ResyncQueuedRowCountAsync(CancellationToken ct) + /// Re-sync both in-memory counters from storage (async path). + private async Task ResyncRowCountsAsync(CancellationToken ct) { Interlocked.Increment(ref _capacityProbeCount); var live = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false); Interlocked.Exchange(ref _queuedRowCount, live); + var dead = await CountAsync("dead_lettered = 1", ct).ConfigureAwait(false); + Interlocked.Exchange(ref _deadLetterRowCount, dead); } private async Task CountAsync(string predicate, CancellationToken ct) @@ -356,6 +373,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl await corruptTx.CommitAsync(ct).ConfigureAwait(false); } Interlocked.Add(ref _queuedRowCount, -corruptIds.Count); + Interlocked.Add(ref _deadLetterRowCount, corruptIds.Count); _logger.Warning( "Dead-lettered {Count} historian queue row(s) with un-deserializable payload", corruptIds.Count); @@ -405,6 +423,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl } int rowsLeavingQueue = 0; + int rowsDeadLettered = 0; await using (var tx = await _db.BeginTransactionAsync(ct).ConfigureAwait(false)) { for (var i = 0; i < outcomes.Count; i++) @@ -423,6 +442,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl case HistorianWriteOutcome.PermanentFail: await DeadLetterRowAsync(tx, id, $"permanent fail at {_clock():O}", ct).ConfigureAwait(false); rowsLeavingQueue++; + rowsDeadLettered++; break; case HistorianWriteOutcome.RetryPlease: // finding 002: cap retries so a perpetually-RetryPlease (poison) @@ -434,6 +454,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl { await DeadLetterRowAsync(tx, id, $"max attempts ({_maxAttempts}) exceeded", ct).ConfigureAwait(false); rowsLeavingQueue++; + rowsDeadLettered++; } else { @@ -449,6 +470,8 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl // cap (finding 002) — keep the counter aligned. if (rowsLeavingQueue > 0) Interlocked.Add(ref _queuedRowCount, -rowsLeavingQueue); + if (rowsDeadLettered > 0) + Interlocked.Add(ref _deadLetterRowCount, rowsDeadLettered); var acks = outcomes.Count(o => o == HistorianWriteOutcome.Ack); lock (_statusLock) @@ -523,13 +546,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl var queued = Interlocked.Read(ref _queuedRowCount); if (queued < 0) queued = 0; - long deadlettered; - using (var conn = _db.CreateConnection()) - using (var cmd = conn.CreateCommand()) - { - cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1"; - deadlettered = (long)(cmd.ExecuteScalar() ?? 0L); - } + // Cached, never a live scan (R05:7) — see the field's remarks for the resync cadence. + var deadlettered = Interlocked.Read(ref _deadLetterRowCount); + if (deadlettered < 0) deadlettered = 0; DateTime? lastDrain, lastSuccess; string? lastError; @@ -568,7 +587,11 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl var revived = cmd.ExecuteNonQuery(); // Dead-lettered rows rejoin the non-dead-lettered queue — keep the in-memory // counter aligned. - if (revived > 0) Interlocked.Add(ref _queuedRowCount, revived); + if (revived > 0) + { + Interlocked.Add(ref _queuedRowCount, revived); + Interlocked.Add(ref _deadLetterRowCount, -revived); + } return revived; } @@ -646,8 +669,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl Interlocked.Increment(ref _capacityProbeCount); var count = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false); - // Resync the in-memory counter while we have a fresh number. + // Resync the in-memory counters while we have a fresh number. Interlocked.Exchange(ref _queuedRowCount, count); + Interlocked.Exchange(ref _deadLetterRowCount, await CountAsync("dead_lettered = 1", ct).ConfigureAwait(false)); if (count < _capacity) return; var toEvict = count - _capacity + 1; @@ -684,7 +708,10 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl ct).ConfigureAwait(false); if (purged > 0) + { + Interlocked.Add(ref _deadLetterRowCount, -purged); _logger.Information("Purged {Count} dead-lettered row(s) past retention window", purged); + } } private void BumpBackoff() => _backoffIndex = Math.Min(_backoffIndex + 1, BackoffLadder.Length - 1); diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index 808935ae..9438132a 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -170,6 +170,24 @@ public sealed class ScriptedAlarmEngine : IDisposable => _scratchByAlarmId.TryGetValue(alarmId, out var s) ? s.Context : null; private readonly ConcurrentDictionary _valueCache = new(StringComparer.Ordinal); + + /// Test hook — the number of paths currently held in the value cache. + internal int ValueCacheCountForTest => _valueCache.Count; + + /// Test hook — whether is currently held in the value cache. + internal bool ValueCacheContainsForTest(string path) => _valueCache.ContainsKey(path); + + /// + /// How long waits for in-flight background work (a re-evaluation or shelving + /// check mid-SaveAsync) before cancelling it through and + /// returning. 5 s, like PollGroupEngine's drain; a hung store must not hang host shutdown. + /// + internal TimeSpan DisposeDrainTimeout { get; set; } = TimeSpan.FromSeconds(5); + + /// Cancelled by once the drain bound expires — every background task + /// (DrainDirtyAsync, ShelvingCheckAsync) runs on its token, so a store call that honours + /// cancellation returns instead of outliving the engine. + private readonly CancellationTokenSource _shutdownCts = new(); private readonly Dictionary> _alarmsReferencing = new(StringComparer.Ordinal); // tag path -> alarm ids @@ -281,6 +299,10 @@ public sealed class ScriptedAlarmEngine : IDisposable UnsubscribeFromUpstream(); _alarms.Clear(); _alarmsReferencing.Clear(); + // The value cache is re-seeded below for exactly the paths this generation references. It + // used to survive reloads untouched, so every path a retired alarm had ever referenced stayed + // cached forever — a slow leak over many deploys with a churning tag set. + _valueCache.Clear(); // Drop the prior generation's per-alarm scratch buffers — definitions may // have changed (different Inputs, different Logger), so any reuse would be // unsafe. @@ -573,8 +595,8 @@ public sealed class ScriptedAlarmEngine : IDisposable /// The new data value snapshot. internal void OnUpstreamChange(string path, DataValueSnapshot value) { + if (_disposed) return; // a disposed engine neither caches nor queues — nothing reads it again _valueCache[path] = value; - if (_disposed) return; // don't queue new work against a disposing engine if (!_alarmsReferencing.TryGetValue(path, out var alarmIds)) return; bool startDrain; @@ -589,7 +611,7 @@ public sealed class ScriptedAlarmEngine : IDisposable // Started outside the lock: DrainDirtyAsync runs synchronously up to its // first real await, and nothing it does should happen under _dirtyLock. - if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(CancellationToken.None)); + if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(_shutdownCts.Token)); } /// @@ -732,6 +754,11 @@ public sealed class ScriptedAlarmEngine : IDisposable } finally { _evalGate.Release(); } } + catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + { + // Dispose cut the drain short after its bound — expected, not a failure. + return; + } catch (Exception ex) { _engineLogger.Error(ex, "ScriptedAlarmEngine reevaluate failed"); @@ -1048,7 +1075,7 @@ public sealed class ScriptedAlarmEngine : IDisposable { if (_disposed) return; var ids = _alarms.Keys.ToArray(); - TrackBackgroundTask(() => ShelvingCheckAsync(ids, CancellationToken.None)); + TrackBackgroundTask(() => ShelvingCheckAsync(ids, _shutdownCts.Token)); } /// @@ -1187,6 +1214,10 @@ public sealed class ScriptedAlarmEngine : IDisposable } finally { _evalGate.Release(); } } + catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + { + return; // Dispose cut the check short after its bound + } catch (Exception ex) { _engineLogger.Warning(ex, "ScriptedAlarmEngine shelving-check failed"); @@ -1227,11 +1258,25 @@ public sealed class ScriptedAlarmEngine : IDisposable // flight can outlive the engine and write to a (possibly disposed) store // after Dispose() has returned. The tasks re-check _disposed after // acquiring the gate and bail out, but the await still has to complete. + // BOUNDED (R05:3): the drained tasks end in IAlarmStateStore.SaveAsync, and a stalled store + // (connection stall, lock, replication hiccup) used to hang this — and the host shutdown that + // calls it — indefinitely. Wait DisposeDrainTimeout; past it, cancel the tasks' shared token (a + // store that honours cancellation returns promptly), log, and give them one short grace period. Task[] toAwait; lock (_inFlightLock) { toAwait = [.. _inFlight]; } if (toAwait.Length > 0) { - try { Task.WhenAll(toAwait).GetAwaiter().GetResult(); } + var all = Task.WhenAll(toAwait); + try { all.WaitAsync(DisposeDrainTimeout).GetAwaiter().GetResult(); } + catch (TimeoutException) + { + _engineLogger.Warning( + "ScriptedAlarmEngine dispose: {Count} background task(s) still running after {Timeout} — cancelling them (a store call is stalled)", + toAwait.Length, DisposeDrainTimeout); + _shutdownCts.Cancel(); + try { all.WaitAsync(TimeSpan.FromSeconds(1)).GetAwaiter().GetResult(); } + catch (Exception) { /* cancelled, faulted, or still stuck — we are leaving either way */ } + } catch (Exception ex) { // Background task failures already logged inside ReevaluateAsync / diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs index dc76923c..9d44aee5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using Serilog; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; @@ -23,19 +24,27 @@ namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; /// adapter doesn't maintain per-subscription state beyond the filter set — it /// checks each emission against every live subscription. /// +/// +/// A throwing subscriber is logged at Warning (naming the alarm) and +/// does not crash the adapter — the same posture as the engine's own FireEvent. It used to +/// be swallowed silently: a lost alarm event with no trace. +/// /// public sealed class ScriptedAlarmSource : IAlarmSource, IDisposable { private readonly ScriptedAlarmEngine _engine; + private readonly ILogger _logger; private readonly ConcurrentDictionary _subscriptions = new(StringComparer.Ordinal); private bool _disposed; /// Initializes a new instance of the class. /// The scripted alarm engine to expose. - public ScriptedAlarmSource(ScriptedAlarmEngine engine) + /// Where a throwing subscriber is reported; null ⇒ Serilog's global logger. + public ScriptedAlarmSource(ScriptedAlarmEngine engine, ILogger? logger = null) { _engine = engine ?? throw new ArgumentNullException(nameof(engine)); + _logger = logger ?? Log.Logger; _engine.OnEvent += OnEngineEvent; } @@ -98,7 +107,13 @@ public sealed class ScriptedAlarmSource : IAlarmSource, IDisposable Severity: evt.Severity, SourceTimestampUtc: evt.TimestampUtc); try { OnAlarmEvent?.Invoke(this, payload); } - catch { /* subscriber exceptions don't crash the adapter */ } + catch (Exception ex) + { + // Subscriber exceptions don't crash the adapter — but they are not silent either: a lost + // alarm event must leave a trace the operator can find. + _logger.Warning(ex, "ScriptedAlarmSource OnAlarmEvent subscriber threw for {AlarmId} ({Emission})", + evt.AlarmId, evt.Emission); + } } } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs index fb1aff35..84bb0e04 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs @@ -24,22 +24,32 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// Original exception (if any) is attached so the main log's diagnostics keep /// the full stack trace. /// +/// +/// Rate-limited per script. A predicate that throws on every push would otherwise mirror +/// one Warning per push into the main log; a keyed by script +/// name admits 20 per minute and mirrors one summary line for the rest when the window rolls. The +/// scripts-*.log file sink is unaffected and keeps the full stream. +/// /// public sealed class ScriptLogCompanionSink : ILogEventSink { private readonly ILogger _mainLogger; private readonly LogEventLevel _minMirrorLevel; + private readonly ScriptLogRateLimiter _limiter; /// Initializes a new instance of the ScriptLogCompanionSink. /// The companion logger to mirror events to. /// The minimum log level to mirror (default: Error). - public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error) + /// Per-script rate limiter; null ⇒ the default 20 per minute. + public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error, + ScriptLogRateLimiter? limiter = null) { _mainLogger = mainLogger ?? throw new ArgumentNullException(nameof(mainLogger)); _minMirrorLevel = minMirrorLevel; + _limiter = limiter ?? new ScriptLogRateLimiter(); } - /// Emits a log event to the companion logger if it meets the minimum level. + /// Emits a log event to the companion logger if it meets the minimum level and the script is within its per-minute budget. /// The log event to emit. public void Emit(LogEvent logEvent) { @@ -54,6 +64,15 @@ public sealed class ScriptLogCompanionSink : ILogEventSink scriptName = s; } + var admission = _limiter.Admit(scriptName); + if (admission.SuppressedInPreviousWindow > 0) + { + _mainLogger.Warning( + "[Script] {ScriptName}: {SuppressedCount} further script error(s) were suppressed in the previous {WindowSeconds:F0} s; see scripts-*.log", + scriptName, admission.SuppressedInPreviousWindow, _limiter.Window.TotalSeconds); + } + if (!admission.Admitted) return; + var rendered = logEvent.RenderMessage(); if (logEvent.Exception is not null) { diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs new file mode 100644 index 00000000..17a6f2f6 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs @@ -0,0 +1,123 @@ +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Per-source fixed-window rate limit for the script-log fan-out: the first +/// entries a source emits in a are +/// admitted, the rest are counted and dropped, and the count is handed back exactly once — on the first +/// admission after the window rolls — so the caller can emit one summary line in the dropped entries' +/// place. Nothing is silently lost (the operator sees "N suppressed"), and one chatty script cannot +/// shed every other script's entries out of the bounded telemetry channels (drop-oldest) or flood the +/// cluster bus / main log. +/// +/// +/// Used by (key = script + virtual-tag + alarm identity), +/// (key = script name) and the virtual-tag actor's per-failure +/// log (one key per tag). Thread-safe; Serilog invokes a sink on whichever thread logged. Idle sources +/// are swept every admissions so a churning tag set cannot grow the table +/// without bound. +/// +public sealed class ScriptLogRateLimiter +{ + /// Entries admitted per source per window by default: 20. + public const int DefaultLimitPerWindow = 20; + + /// The default window: one minute. + public static readonly TimeSpan DefaultWindow = TimeSpan.FromMinutes(1); + + private const int SweepEvery = 256; + + private readonly Dictionary _sources = new(StringComparer.Ordinal); + private readonly int _limit; + private readonly TimeSpan _window; + private readonly Func _clock; + private int _admitsSinceSweep; + + /// Initializes a limiter. + /// Entries admitted per source per window; values below 1 are treated as 1. + /// The fixed window; null ⇒ . + /// UTC clock; null ⇒ . + public ScriptLogRateLimiter(int limitPerWindow = DefaultLimitPerWindow, TimeSpan? window = null, Func? clock = null) + { + _limit = Math.Max(1, limitPerWindow); + _window = window is { } w && w > TimeSpan.Zero ? w : DefaultWindow; + _clock = clock ?? (static () => DateTime.UtcNow); + } + + /// The window length. + public TimeSpan Window => _window; + + /// + /// Ask whether 's next entry may be emitted. When the window has rolled + /// since the last call, carries the + /// number of entries dropped in the window that just closed (zero otherwise) — the caller emits one + /// summary for them BEFORE the admitted entry. + /// + /// Identity of the emitting script instance. + /// The admission decision. + public ScriptLogAdmission Admit(string sourceKey) + { + ArgumentNullException.ThrowIfNull(sourceKey); + var now = _clock(); + lock (_sources) + { + if (++_admitsSinceSweep >= SweepEvery) + { + _admitsSinceSweep = 0; + Sweep(now); + } + + if (!_sources.TryGetValue(sourceKey, out var source)) + { + source = new Source { WindowStart = now }; + _sources[sourceKey] = source; + } + + long summary = 0; + if (now - source.WindowStart >= _window) + { + summary = source.Suppressed; + source.WindowStart = now; + source.Admitted = 0; + source.Suppressed = 0; + } + + source.LastSeen = now; + if (source.Admitted < _limit) + { + source.Admitted++; + return new ScriptLogAdmission(true, summary); + } + + source.Suppressed++; + return new ScriptLogAdmission(false, summary); + } + } + + private void Sweep(DateTime now) + { + var idle = now - (_window + _window); + List? dead = null; + foreach (var (key, source) in _sources) + { + // A source that still has a pending summary is kept: its next entry must report the count. + if (source.LastSeen < idle && source.Suppressed == 0) + (dead ??= []).Add(key); + } + if (dead is null) return; + foreach (var key in dead) _sources.Remove(key); + } + + private sealed class Source + { + public DateTime WindowStart; + public DateTime LastSeen; + public int Admitted; + public long Suppressed; + } +} + +/// Result of . +/// Whether the entry may be emitted. +/// Entries dropped in the window that just closed — non-zero at +/// most once per window roll; the caller emits one summary line for them. +public readonly record struct ScriptLogAdmission(bool Admitted, long SuppressedInPreviousWindow); diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs index fe9e7f3e..a7b8f03e 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs @@ -25,25 +25,37 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// property so pipelines that pre-date this sink continue to work correctly. If /// neither property is present ScriptId defaults to "unknown". /// +/// +/// Rate-limited per source. A script that logs at Information on every evaluation of a 1 Hz +/// input put one rendered string per second per script onto the cluster bus; 200 such scripts shed +/// everyone's entries out of the bounded telemetry channels. A +/// keyed by the entry's identity (script + virtual tag + alarm) admits 20 entries per minute per +/// source and publishes one summary entry naming the dropped count when the window rolls. +/// /// public sealed class ScriptLogTopicSink : ILogEventSink { private readonly IScriptLogPublisher _publisher; private readonly LogEventLevel _min; + private readonly ScriptLogRateLimiter _limiter; /// Initializes a new instance of the class. /// The publisher that routes entries to the cluster bus. Must not be null. /// Minimum log level to forward. Events below this level are discarded (default: ). + /// Per-source rate limiter; null ⇒ the default 20 per minute. /// Thrown when is null. - public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information) + public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information, + ScriptLogRateLimiter? limiter = null) { _publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); _min = min; + _limiter = limiter ?? new ScriptLogRateLimiter(); } /// /// Converts the to a and - /// publishes it, unless the event is below the configured minimum level. + /// publishes it, unless the event is below the configured minimum level or the source is over its + /// per-minute budget. /// /// The Serilog event to process. Silently ignored when null. public void Emit(LogEvent logEvent) @@ -56,15 +68,36 @@ public sealed class ScriptLogTopicSink : ILogEventSink ? s : null; + var scriptId = P(ScriptLoggerFactory.ScriptIdProperty) + ?? P(ScriptLoggerFactory.ScriptNameProperty) + ?? "unknown"; + var virtualTagId = P(ScriptLoggerFactory.VirtualTagIdProperty); + var alarmId = P(ScriptLoggerFactory.AlarmIdProperty); + var equipmentId = P(ScriptLoggerFactory.EquipmentIdProperty); + + var admission = _limiter.Admit(string.Concat(scriptId, "|", virtualTagId, "|", alarmId)); + if (admission.SuppressedInPreviousWindow > 0) + { + _publisher.Publish(new ScriptLogEntry( + ScriptId: scriptId, + Level: nameof(LogEventLevel.Warning), + Message: $"{admission.SuppressedInPreviousWindow} script-log entr{(admission.SuppressedInPreviousWindow == 1 ? "y" : "ies")} " + + $"from this source were suppressed in the previous {_limiter.Window.TotalSeconds:F0} s " + + $"(limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); see scripts-*.log for the full stream", + TimestampUtc: logEvent.Timestamp.UtcDateTime, + VirtualTagId: virtualTagId, + AlarmId: alarmId, + EquipmentId: equipmentId)); + } + if (!admission.Admitted) return; + _publisher.Publish(new ScriptLogEntry( - ScriptId: P(ScriptLoggerFactory.ScriptIdProperty) - ?? P(ScriptLoggerFactory.ScriptNameProperty) - ?? "unknown", + ScriptId: scriptId, Level: logEvent.Level.ToString(), Message: logEvent.RenderMessage(), TimestampUtc: logEvent.Timestamp.UtcDateTime, - VirtualTagId: P(ScriptLoggerFactory.VirtualTagIdProperty), - AlarmId: P(ScriptLoggerFactory.AlarmIdProperty), - EquipmentId: P(ScriptLoggerFactory.EquipmentIdProperty))); + VirtualTagId: virtualTagId, + AlarmId: alarmId, + EquipmentId: equipmentId)); } } diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs index 9c4c6b11..97d68679 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs @@ -342,6 +342,40 @@ public sealed class LocalDbStoreAndForwardSinkTests : IDisposable sink.GetStatus().DeadLetterDepth.ShouldBe(0, "purged past retention"); } + /// + /// Review R05:7 — GetStatus no longer runs a COUNT(*) table scan per call for the + /// dead-letter depth: it reads a cached counter kept in step by every dead-letter / revive / purge + /// path (the other tests in this class pin those), and resynced from storage on the same cadence as + /// the queued-row counter. A row flipped underneath the sink (the peer replicating a dead-letter in) + /// is therefore invisible until the next resync — that is the documented trade, and the resync is + /// what makes it bounded rather than permanent. + /// + [Fact] + public async Task DeadLetterDepth_is_served_from_a_cached_counter_that_the_periodic_resync_corrects() + { + var writer = new FakeWriter(); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); + await sink.EnqueueAsync(Event("A1"), CancellationToken.None); + await sink.EnqueueAsync(Event("A2"), CancellationToken.None); + sink.GetStatus().DeadLetterDepth.ShouldBe(0); + var probesBefore = sink.DebugCapacityProbeCount; + + // Drift the table underneath the sink, exactly as a replicated dead-letter from the pair peer would. + await Db.ExecuteAsync( + "UPDATE alarm_sf_events SET dead_lettered = 1 WHERE id IN (SELECT id FROM alarm_sf_events LIMIT 1)", + null, CancellationToken.None); + + // Cached: GetStatus did not go to the table, so it still reports the pre-drift depth… + sink.GetStatus().DeadLetterDepth.ShouldBe(0, "GetStatus serves the cached counter, not a live COUNT(*)"); + sink.DebugCapacityProbeCount.ShouldBe(probesBefore, "GetStatus must not probe storage"); + + // …and the resync (the one the enqueue cadence drives) brings both counters back in line. + await sink.DebugResyncCountersAsync(CancellationToken.None); + var status = sink.GetStatus(); + status.DeadLetterDepth.ShouldBe(1); + status.QueueDepth.ShouldBe(1); + } + /// Verifies that RetryDeadLettered requeues dead-lettered rows for retry. [Fact] public async Task RetryDeadLettered_requeues_for_retry() diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs new file mode 100644 index 00000000..7afea07a --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs @@ -0,0 +1,159 @@ +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; + +/// +/// Review R05:1 / R05:12 / R05:3 — the engine's value cache is pruned on every reload (it used to keep +/// every path ever referenced, forever), is no longer written after Dispose, and Dispose's in-flight +/// drain is bounded (a hung store no longer hangs host shutdown). +/// +[Trait("Category", "Unit")] +public sealed class ScriptedAlarmEngineLifecycleTests +{ + private static ScriptedAlarmDefinition Alarm(string id, string predicate) => + new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, + Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); + + private static ScriptedAlarmEngine Build(FakeUpstream up, IAlarmStateStore? store = null) + { + var logger = new LoggerConfiguration().CreateLogger(); + return new ScriptedAlarmEngine(up, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger); + } + + [Fact] + public async Task Reload_prunes_value_cache_entries_for_paths_the_new_alarm_set_no_longer_references() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + up.Set("Pressure", 5); + using var eng = Build(up); + var ct = TestContext.Current.CancellationToken; + + await eng.LoadAsync( + [Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100 || (int)ctx.GetTag("Pressure").Value > 10;""")], ct); + eng.ValueCacheCountForTest.ShouldBe(2); + + // The redeploy drops Pressure from every predicate: its entry must go with it. + await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], ct); + eng.ValueCacheCountForTest.ShouldBe(1); + eng.ValueCacheContainsForTest("Pressure").ShouldBeFalse(); + eng.ValueCacheContainsForTest("Temp").ShouldBeTrue(); + } + + [Fact] + public async Task A_push_after_Dispose_does_not_write_the_value_cache() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + var eng = Build(up); + await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + eng.Dispose(); + eng.ValueCacheCountForTest.ShouldBe(1); + + eng.OnUpstreamChange("Late", new DataValueSnapshot(1, 0u, DateTime.UtcNow, DateTime.UtcNow)); + + eng.ValueCacheContainsForTest("Late").ShouldBeFalse("a disposed engine must not grow its cache"); + } + + [Fact] + public async Task Dispose_gives_up_on_a_store_that_never_returns_after_the_drain_bound_and_cancels_it() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + var store = new NeverReturningSaveStore(); + var eng = Build(up, store); + eng.DisposeDrainTimeout = TimeSpan.FromMilliseconds(300); + await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + + store.BlockNext = true; + up.Push("Temp", 150); // Activated → SaveAsync → blocks forever (until cancelled) + await WaitForAsync(() => store.SaveInProgress); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + eng.Dispose(); + sw.Stop(); + + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the drain is bounded; a hung store cannot hang shutdown"); + await WaitForAsync(() => store.ObservedCancellation, timeoutMs: 2000); + store.ObservedCancellation.ShouldBeTrue("the in-flight save was cancelled through the engine's shutdown token"); + } + + [Fact] + public async Task Dispose_still_waits_for_a_save_that_completes_inside_the_bound() + { + // Positive control for the bound: a save that finishes promptly is drained, not cut off. + var up = new FakeUpstream(); + up.Set("Temp", 50); + var store = new NeverReturningSaveStore { ReleaseAfter = TimeSpan.FromMilliseconds(100) }; + var eng = Build(up, store); + eng.DisposeDrainTimeout = TimeSpan.FromSeconds(5); + await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + + store.BlockNext = true; + up.Push("Temp", 150); + await WaitForAsync(() => store.SaveInProgress); + + eng.Dispose(); + + store.SaveInProgress.ShouldBeFalse(); + store.ObservedCancellation.ShouldBeFalse("a save that completes within the bound is never cancelled"); + store.Saved.ShouldContain("A"); + } + + private static async Task WaitForAsync(Func cond, int timeoutMs = 5000) + { + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs); + while (DateTime.UtcNow < deadline) + { + if (cond()) return; + await Task.Delay(20); + } + throw new TimeoutException("Condition did not become true in time"); + } + + /// A store whose next SaveAsync blocks until cancelled (or until ). + private sealed class NeverReturningSaveStore : IAlarmStateStore + { + private readonly InMemoryAlarmStateStore _inner = new(); + public volatile bool BlockNext; + public volatile bool SaveInProgress; + public volatile bool ObservedCancellation; + public TimeSpan? ReleaseAfter { get; set; } + public List Saved { get; } = []; + + public Task LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct); + public Task> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct); + public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct); + + public async Task SaveAsync(AlarmConditionState state, CancellationToken ct) + { + if (BlockNext) + { + BlockNext = false; + SaveInProgress = true; + try + { + await Task.Delay(ReleaseAfter ?? Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + ObservedCancellation = true; + throw; + } + finally + { + SaveInProgress = false; + } + } + lock (Saved) Saved.Add(state.AlarmId); + await _inner.SaveAsync(state, ct).ConfigureAwait(false); + } + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs new file mode 100644 index 00000000..94c0408c --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs @@ -0,0 +1,59 @@ +using Serilog; +using Serilog.Core; +using Serilog.Events; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; + +/// Review R05:6 — a throwing subscriber used to be +/// swallowed with no diagnostic: a lost alarm event with no trace. It is now logged at Warning. +[Trait("Category", "Unit")] +public sealed class ScriptedAlarmSourceLoggingTests +{ + private sealed class CapturingSink : ILogEventSink + { + public List Events { get; } = []; + public void Emit(LogEvent logEvent) { lock (Events) Events.Add(logEvent); } + } + + [Fact] + public async Task A_throwing_subscriber_is_logged_at_Warning_and_does_not_crash_the_adapter() + { + var sinkCapture = new CapturingSink(); + var logger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sinkCapture).CreateLogger(); + var up = new FakeUpstream(); + up.Set("Temp", 50); + var engine = new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger); + await engine.LoadAsync( + [new ScriptedAlarmDefinition("Plant/Line1::HighTemp", "Plant/Line1", "HighTemp", AlarmKind.LimitAlarm, + AlarmSeverity.High, "Temp {Temp}C", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + using var _e = engine; + using var source = new ScriptedAlarmSource(engine, logger); + await source.SubscribeAlarmsAsync(Array.Empty(), TestContext.Current.CancellationToken); + source.OnAlarmEvent += (_, _) => throw new InvalidOperationException("subscriber broke"); + + up.Push("Temp", 150); + + var deadline = DateTime.UtcNow.AddSeconds(10); + while (DateTime.UtcNow < deadline) + { + lock (sinkCapture.Events) + { + if (sinkCapture.Events.Any(e => e.Level == LogEventLevel.Warning && e.Exception?.Message == "subscriber broke")) + break; + } + await Task.Delay(25); + } + + LogEvent warning; + lock (sinkCapture.Events) + warning = sinkCapture.Events.Single(e => e.Level == LogEventLevel.Warning && e.Exception?.Message == "subscriber broke"); + warning.RenderMessage().ShouldContain("Plant/Line1::HighTemp"); + // The adapter survived: the engine's state still moved. + engine.GetState("Plant/Line1::HighTemp")!.Active.ShouldBe(AlarmActiveState.Active); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs new file mode 100644 index 00000000..3f560e40 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs @@ -0,0 +1,137 @@ +using Serilog; +using Serilog.Core; +using Serilog.Events; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; + +/// +/// Per-source rate limit on the script-log fan-out (review R05:9 / R02:P6): a script that logs — or +/// fails — on every evaluation of a 1 Hz input used to put one entry per second per script onto the +/// cluster bus and the main log. The limiter admits the first 20 entries per source per minute, drops +/// the rest, and surfaces one summary line (with the dropped count) when the next entry arrives after +/// the window rolled — so nothing is silently lost, and a chatty script cannot shed everyone else's +/// entries out of the bounded telemetry channels. +/// +[Trait("Category", "Unit")] +public sealed class ScriptLogRateLimiterTests +{ + private sealed class Clock + { + public DateTime Now = new(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc); + public void Advance(TimeSpan by) => Now += by; + } + + [Fact] + public void Admits_the_first_N_per_window_then_suppresses_and_summarises_once_the_window_rolls() + { + var clock = new Clock(); + var limiter = new ScriptLogRateLimiter(limitPerWindow: 3, window: TimeSpan.FromMinutes(1), clock: () => clock.Now); + + limiter.Admit("s1").Admitted.ShouldBeTrue(); + limiter.Admit("s1").Admitted.ShouldBeTrue(); + limiter.Admit("s1").Admitted.ShouldBeTrue(); + limiter.Admit("s1").Admitted.ShouldBeFalse(); + limiter.Admit("s1").Admitted.ShouldBeFalse(); + limiter.Admit("s1").SuppressedInPreviousWindow.ShouldBe(0, "no summary until the window rolls"); + + clock.Advance(TimeSpan.FromMinutes(1)); + var first = limiter.Admit("s1"); + first.Admitted.ShouldBeTrue("a fresh window admits again"); + first.SuppressedInPreviousWindow.ShouldBe(3, "the caller emits one summary for the three dropped entries"); + limiter.Admit("s1").SuppressedInPreviousWindow.ShouldBe(0, "the summary is reported exactly once"); + } + + [Fact] + public void Sources_are_independent() + { + var limiter = new ScriptLogRateLimiter(limitPerWindow: 1); + limiter.Admit("a").Admitted.ShouldBeTrue(); + limiter.Admit("a").Admitted.ShouldBeFalse(); + limiter.Admit("b").Admitted.ShouldBeTrue("a chatty source must not shed another source's entries"); + } + + [Fact] + public void Defaults_are_20_per_minute() + { + ScriptLogRateLimiter.DefaultLimitPerWindow.ShouldBe(20); + ScriptLogRateLimiter.DefaultWindow.ShouldBe(TimeSpan.FromMinutes(1)); + var limiter = new ScriptLogRateLimiter(); + for (var i = 0; i < 20; i++) limiter.Admit("s").Admitted.ShouldBeTrue(); + limiter.Admit("s").Admitted.ShouldBeFalse(); + } + + // ── the two sinks ──────────────────────────────────────────────────────────────────────── + + private sealed class FakePublisher : IScriptLogPublisher + { + public List Published { get; } = []; + public void Publish(ScriptLogEntry entry) => Published.Add(entry); + } + + private static LogEvent Evt(LogEventLevel level, string template, params (string Key, string Val)[] props) + { + var tmpl = new Serilog.Parsing.MessageTemplateParser().Parse(template); + var logProps = props.Select(p => new LogEventProperty(p.Key, new ScalarValue(p.Val))); + return new LogEvent(DateTimeOffset.UtcNow, level, exception: null, tmpl, logProps); + } + + [Fact] + public void Topic_sink_rate_limits_per_source_and_publishes_a_summary_entry() + { + var clock = new Clock(); + var publisher = new FakePublisher(); + var sink = new ScriptLogTopicSink(publisher, LogEventLevel.Information, + new ScriptLogRateLimiter(limitPerWindow: 2, clock: () => clock.Now)); + + for (var i = 0; i < 5; i++) + sink.Emit(Evt(LogEventLevel.Warning, "chatty", (ScriptLoggerFactory.ScriptIdProperty, "S1"), (ScriptLoggerFactory.VirtualTagIdProperty, "V1"))); + sink.Emit(Evt(LogEventLevel.Warning, "other", (ScriptLoggerFactory.ScriptIdProperty, "S2"))); + + publisher.Published.Count(e => e.ScriptId == "S1").ShouldBe(2, "the limit is per source"); + publisher.Published.Count(e => e.ScriptId == "S2").ShouldBe(1, "another source is unaffected"); + + clock.Advance(TimeSpan.FromMinutes(1)); + sink.Emit(Evt(LogEventLevel.Warning, "chatty again", (ScriptLoggerFactory.ScriptIdProperty, "S1"), (ScriptLoggerFactory.VirtualTagIdProperty, "V1"))); + + var s1 = publisher.Published.Where(e => e.ScriptId == "S1").ToList(); + s1.Count.ShouldBe(4, "2 admitted + 1 summary + 1 admitted in the new window"); + var summary = s1[2]; + summary.Message.ShouldContain("3"); + summary.Message.ShouldContain("suppressed"); + summary.VirtualTagId.ShouldBe("V1", "the summary carries the source's identity so the page attributes it"); + summary.Level.ShouldBe("Warning"); + s1[3].Message.ShouldBe("chatty again"); + } + + private sealed class CapturingSink : ILogEventSink + { + public List Events { get; } = []; + public void Emit(LogEvent logEvent) => Events.Add(logEvent); + } + + [Fact] + public void Companion_sink_rate_limits_per_script_and_mirrors_a_summary_line() + { + var clock = new Clock(); + var main = new CapturingSink(); + var mainLogger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(main).CreateLogger(); + var sink = new ScriptLogCompanionSink(mainLogger, LogEventLevel.Error, + new ScriptLogRateLimiter(limitPerWindow: 2, clock: () => clock.Now)); + + for (var i = 0; i < 5; i++) + sink.Emit(Evt(LogEventLevel.Error, "boom", (ScriptLoggerFactory.ScriptNameProperty, "HighTemp"))); + main.Events.Count.ShouldBe(2); + + clock.Advance(TimeSpan.FromMinutes(1)); + sink.Emit(Evt(LogEventLevel.Error, "boom", (ScriptLoggerFactory.ScriptNameProperty, "HighTemp"))); + + main.Events.Count.ShouldBe(4); + var summary = main.Events[2].RenderMessage(); + summary.ShouldContain("HighTemp"); + summary.ShouldContain("3"); + summary.ShouldContain("suppressed"); + } +} From f72af76e99e241ad1d36b90a8d0952c7d74a17d1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:23:14 -0400 Subject: [PATCH 08/27] fix(security): validate the API-key pepper at start, finish JWT bearer, circuit-aware audit actor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ApiKeyOptionsValidator (ValidateOnStart on any driver/admin node): a blank or placeholder Security:ApiKeys:Pepper fails host start instead of rejecting every key after a green boot (R03:2.3). - JWT is finished and opt-in by key (R03:2.4): with Security:Jwt:SigningKey set, a Bearer scheme (JwtBearerAuthenticationHandler over JwtTokenService.TryValidate / BuildValidationParameters, fail closed) accepts the tokens /auth/token mints; the default scheme is a cookie-or-bearer policy scheme and the fallback policy names it, so /api/* and every RequireAuthorization endpoint take a browser session OR a machine token. JwtOptions are validated at start (JwtOptionsValidator). Without a key, /auth/token is not mapped (404, not the former constructor 500) and no bearer scheme exists. A direct AddJwtBearer needs Microsoft.AspNetCore.Authentication.JwtBearer in Directory.Packages.props (not this lane's file) — the handler is a drop-in to swap. - AuthenticationStateAuditActorAccessor is the registered IAuditActorAccessor: the Blazor circuit's principal first, HttpContext as the fallback (R03:2.5). HttpAuditActorAccessor shares its resolver. - CookieAuthenticationStateProvider (unregistered, fire-and-forget ping loop) is deleted (R03:1.5/2.5). - /auth/login logs the LDAP exception at Warning before answering 503 (R03:2.7). - Admin operations reply with a fixed message + correlation id instead of ex.Message (R03:4.1). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdminOperations/AdminOperationsActor.cs | 40 ++++-- .../Configuration/ApiKeyOptionsValidator.cs | 41 ++++++ src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 13 +- .../AuthenticationStateAuditActorAccessor.cs | 75 +++++++++++ .../Audit/HttpAuditActorAccessor.cs | 26 +--- .../Auth/JwtBearerAuthenticationHandler.cs | 99 ++++++++++++++ .../CookieAuthenticationStateProvider.cs | 79 ----------- .../Endpoints/AuthEndpoints.cs | 17 ++- .../Jwt/JwtOptions.cs | 22 +++ .../Jwt/JwtOptionsValidator.cs | 19 +++ .../ServiceCollectionExtensions.cs | 48 +++++-- ...nOperationsActorServerFaultMessageTests.cs | 68 ++++++++++ .../ApiKeyOptionsValidatorTests.cs | 54 ++++++++ ...henticationStateAuditActorAccessorTests.cs | 80 +++++++++++ .../AuthEndpointsIntegrationTests.cs | 67 +++++++++ .../CapturingLoggerProvider.cs | 25 ++++ .../JwtConfigurationGateTests.cs | 127 ++++++++++++++++++ 17 files changed, 773 insertions(+), 127 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs delete mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index 8b1c9ddb..e78c926e 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -89,6 +89,18 @@ public sealed class AdminOperationsActor : ReceiveActor Receive(HandleShelveAlarm); } + /// + /// The reply text for an unexpected server-side failure (R03:4.1): a fixed sentence plus the request's + /// correlation id, never ex.Message — a SqlException or UriFormatException message carries + /// connection-string fragments and host names, and the reply reaches any ConfigEditor browser session. + /// The detail is in the central log under the same correlation id. + /// + /// The admin operation that failed. + /// The request's correlation token, as logged alongside the exception. + /// The operator-facing message. + internal static string ServerFault(string operation, object correlationId) => + $"{operation} failed on the server; see the central node log for correlation {correlationId}."; + /// /// Hands a central→node command to the mesh transport router, or publishes it directly when /// no router is wired. @@ -136,8 +148,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId}", msg.AlarmId); - replyTo.Tell(new AcknowledgeAlarmResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId} (correlation {CorrelationId})", msg.AlarmId, msg.CorrelationId); + replyTo.Tell(new AcknowledgeAlarmResult(false, ServerFault("Acknowledge", msg.CorrelationId), msg.CorrelationId)); } } @@ -183,8 +195,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId}", msg.Kind, msg.AlarmId); - replyTo.Tell(new ShelveAlarmResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId} (correlation {CorrelationId})", msg.Kind, msg.AlarmId, msg.CorrelationId); + replyTo.Tell(new ShelveAlarmResult(false, ServerFault("Shelve", msg.CorrelationId), msg.CorrelationId)); } } @@ -237,12 +249,12 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "StartDeployment failed for {CreatedBy}", msg.CreatedBy); + _log.Error(ex, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.CreatedBy, msg.CorrelationId); replyTo.Tell(new StartDeploymentResult( StartDeploymentOutcome.Rejected, DeploymentId: null, RevisionHash: null, - Message: ex.Message, + Message: ServerFault("StartDeployment", msg.CorrelationId), msg.CorrelationId)); return; } @@ -269,12 +281,12 @@ public sealed class AdminOperationsActor : ReceiveActor private void HandleDeployComposeFailed(DeployComposeFailed msg) { _composeInFlight = null; - _log.Error(msg.Error, "StartDeployment failed for {CreatedBy}", msg.Request.CreatedBy); + _log.Error(msg.Error, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.Request.CreatedBy, msg.Request.CorrelationId); msg.ReplyTo.Tell(new StartDeploymentResult( StartDeploymentOutcome.Rejected, DeploymentId: null, RevisionHash: null, - Message: msg.Error.Message, + Message: ServerFault("StartDeployment", msg.Request.CorrelationId), msg.Request.CorrelationId)); } @@ -513,10 +525,10 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "Probe for {DriverType} threw", msg.DriverType); + _log.Error(ex, "Probe for {DriverType} threw (correlation {CorrelationId})", msg.DriverType, msg.CorrelationId); return new TestDriverConnectResult( false, - ex.Message, + ServerFault("Test Connect", msg.CorrelationId), null, msg.CorrelationId); } @@ -548,8 +560,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId}", msg.DriverInstanceId); - replyTo.Tell(new RestartDriverResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId); + replyTo.Tell(new RestartDriverResult(false, ServerFault("RestartDriver", msg.CorrelationId), msg.CorrelationId)); } } @@ -578,8 +590,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId}", msg.DriverInstanceId); - replyTo.Tell(new ReconnectDriverResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId); + replyTo.Tell(new ReconnectDriverResult(false, ServerFault("ReconnectDriver", msg.CorrelationId), msg.CorrelationId)); } } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs new file mode 100644 index 00000000..eced66a8 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs @@ -0,0 +1,41 @@ +using ZB.MOM.WW.Configuration; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; + +namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; + +/// +/// Fail-fast startup validator for (R03:2.3). The pepper used to be +/// checked only when the first key was verified — ApiKeySecret.Verify throws on an empty one — +/// so a site node deployed without Security__ApiKeys__Pepper booted green, passed health, +/// applied deployments and denied every machine client with the same opaque "Invalid API key" until +/// someone read the log; a central node with a wrong pepper minted keys that could verify nowhere. +/// Every other security section already fails host start on a bad value; this one now does too. +/// +/// +/// Registered for hasDriver || hasAdmin: a driver node needs the pepper to VERIFY keys, an admin +/// node to MINT them. The minimum length is a floor against a placeholder ("x") passing as +/// configured, not a strength claim — the pepper is a server-side secret, not a password. +/// +public sealed class ApiKeyOptionsValidator : OptionsValidatorBase +{ + /// The shortest pepper accepted (characters, after trimming). + public const int MinPepperLength = 16; + + /// + protected override void Validate(ValidationBuilder builder, ApiKeyOptions options) + { + var pepper = options.Pepper?.Trim() ?? string.Empty; + builder.RequireThat(pepper.Length > 0, + $"{ApiKeyOptions.SectionName}:Pepper is required on any node with the driver or admin role — " + + "supply it via Security__ApiKeys__Pepper or a ${secret:} reference. An empty pepper would reject " + + "every API key (driver) or mint keys that can never verify (admin)."); + builder.RequireThat(pepper.Length == 0 || pepper.Length >= MinPepperLength, + $"{ApiKeyOptions.SectionName}:Pepper must be at least {MinPepperLength} characters."); + builder.RequireThat(!string.IsNullOrWhiteSpace(options.Username), + $"{ApiKeyOptions.SectionName}:Username must not be blank."); + builder.RequireThat(options.StoreTimeoutMs >= 0, + $"{ApiKeyOptions.SectionName}:StoreTimeoutMs must be >= 0 (0 disables the deadline)."); + builder.RequireThat(options.StoreCacheSeconds >= 0, + $"{ApiKeyOptions.SectionName}:StoreCacheSeconds must be >= 0 (0 disables the cache)."); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index d326d1b1..b2300ab6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -162,10 +162,15 @@ builder.AddZbSerilog(o => o.ServiceName = "otopcua"); // central over the fetch-and-cache path (ConfigSource:Mode=FetchAndCache). Registering it // unconditionally would throw on a driver-only node, since AddOtOpcUaConfigDb requires the string. // A fused admin+driver node keeps ConfigDb via its admin role — byte-for-byte unaffected. -// bound unconditionally: a driver node needs the pepper to VERIFY keys, an admin node needs it -// to MINT them, and getting this wrong on either side is a silent authentication failure rather than a -// startup error. -builder.Services.Configure(builder.Configuration.GetSection(ApiKeyOptions.SectionName)); +// A driver node needs the pepper to VERIFY keys, an admin node needs it to MINT them, and getting this +// wrong on either side used to be a silent authentication failure rather than a startup error (R03:2.3): +// ApiKeyOptionsValidator now fails host start on a blank or placeholder pepper wherever either role is +// present. A node with neither role (dev-only) has no key path and keeps the plain bind. +if (hasDriver || hasAdmin) + builder.Services.AddValidatedOptions( + builder.Configuration, ApiKeyOptions.SectionName); +else + builder.Services.Configure(builder.Configuration.GetSection(ApiKeyOptions.SectionName)); // Security:Acl:Enabled. Default false; see AclOptions for why the flip must be a config // change rather than a redeploy, and what it swaps rather than adds. builder.Services.Configure(builder.Configuration.GetSection(AclOptions.SectionName)); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs new file mode 100644 index 00000000..d689b85e --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs @@ -0,0 +1,75 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Components.Authorization; +using Microsoft.AspNetCore.Http; +using ZB.MOM.WW.Auth.AspNetCore; + +namespace ZB.MOM.WW.OtOpcUa.Security.Audit; + +/// +/// that reads the Blazor circuit's principal first (R03:2.5). +/// +/// +/// +/// reads IHttpContextAccessor, which is right for a +/// minimal-API request but wrong inside a Blazor Server circuit: after the first render there is no +/// current HttpContext on the circuit's threads, so every audit row written from an interactive +/// page would have been stamped "system". The circuit's identity lives in the scoped +/// , so that is consulted first and the HTTP context is the +/// fallback for non-circuit scopes (where the provider is not registered at all). +/// +/// +/// CurrentActor is synchronous by contract; the server-side provider's +/// is already completed in a +/// circuit, and an incomplete task is treated as "no circuit identity" rather than blocked on. +/// +/// +public sealed class AuthenticationStateAuditActorAccessor : IAuditActorAccessor +{ + private readonly IHttpContextAccessor _httpContextAccessor; + private readonly AuthenticationStateProvider? _authenticationState; + + /// Initializes a new instance of the class. + /// The HTTP context accessor (minimal-API fallback). + /// The circuit's authentication state, when resolved inside a Blazor circuit. + public AuthenticationStateAuditActorAccessor( + IHttpContextAccessor httpContextAccessor, + AuthenticationStateProvider? authenticationState = null) + { + ArgumentNullException.ThrowIfNull(httpContextAccessor); + _httpContextAccessor = httpContextAccessor; + _authenticationState = authenticationState; + } + + /// + public string? CurrentActor + { + get + { + if (_authenticationState is not null) + { + var task = _authenticationState.GetAuthenticationStateAsync(); + if (task.IsCompletedSuccessfully) + { + var actor = ResolveActor(task.Result.User); + if (actor is not null) return actor; + } + } + return ResolveActor(_httpContextAccessor.HttpContext?.User); + } + } + + /// The claim-resolution rule shared with . + /// The principal, or null. + /// The actor string, or null when unauthenticated. + internal static string? ResolveActor(ClaimsPrincipal? user) + { + if (user?.Identity?.IsAuthenticated != true) + return null; + + // Prefer the canonical login-name claim; fall back to the Name claim or + // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name). + return user.FindFirst(ZbClaimTypes.Username)?.Value + ?? user.FindFirst(ZbClaimTypes.Name)?.Value + ?? user.Identity.Name; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs index 9a86339f..529504b3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs @@ -1,5 +1,4 @@ using Microsoft.AspNetCore.Http; -using ZB.MOM.WW.Auth.AspNetCore; namespace ZB.MOM.WW.OtOpcUa.Security.Audit; @@ -17,10 +16,10 @@ namespace ZB.MOM.WW.OtOpcUa.Security.Audit; /// , in that order. /// /// -/// Registered as scoped in -/// so that it correctly follows the request scope used by Blazor Server interactive components -/// and minimal-API endpoints. IHttpContextAccessor is registered by -/// AddOtOpcUaAuth via services.AddHttpContextAccessor(). +/// No longer the registered implementation: is, because +/// a Blazor circuit has no HttpContext after its first render and this accessor would stamp every row +/// written from an interactive page "system". Kept as the explicit HTTP-only variant for non-circuit +/// hosts and tests. /// /// public sealed class HttpAuditActorAccessor : IAuditActorAccessor @@ -35,19 +34,6 @@ public sealed class HttpAuditActorAccessor : IAuditActorAccessor } /// - public string? CurrentActor - { - get - { - var user = _httpContextAccessor.HttpContext?.User; - if (user?.Identity?.IsAuthenticated != true) - return null; - - // Prefer the canonical login-name claim; fall back to the Name claim or - // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name). - return user.FindFirst(ZbClaimTypes.Username)?.Value - ?? user.FindFirst(ZbClaimTypes.Name)?.Value - ?? user.Identity.Name; - } - } + public string? CurrentActor => + AuthenticationStateAuditActorAccessor.ResolveActor(_httpContextAccessor.HttpContext?.User); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs new file mode 100644 index 00000000..cc996758 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs @@ -0,0 +1,99 @@ +using System.Text.Encodings.Web; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Primitives; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; + +namespace ZB.MOM.WW.OtOpcUa.Security.Auth; + +/// The authentication scheme names OtOpcUa registers. +public static class OtOpcUaAuthSchemes +{ + /// + /// The bearer scheme that validates the HS256 tokens /auth/token mints, through + /// . Registered only when + /// . + /// + public const string Bearer = "Bearer"; + + /// + /// The default scheme: a policy scheme that forwards to when the request + /// carries an Authorization: Bearer header and to the cookie scheme otherwise, so one + /// RequireAuthorization() / policy accepts a browser session AND a machine token. + /// + public const string CookieOrBearer = "OtOpcUa.CookieOrBearer"; + + /// Whether the request presents a bearer token (the selector's whole rule). + /// The request. + /// when an Authorization: Bearer … header is present. + public static bool HasBearerToken(HttpContext context) => + TryGetBearerToken(context.Request.Headers.Authorization, out _); + + internal static bool TryGetBearerToken(StringValues authorization, out string token) + { + token = string.Empty; + var header = authorization.ToString(); + const string prefix = "Bearer "; + if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; + token = header[prefix.Length..].Trim(); + return token.Length > 0; + } +} + +/// +/// Validates an Authorization: Bearer token minted by /auth/token (R03:2.4). The JWT +/// feature used to be a dead end: tokens were issued and nothing in the product accepted them. This +/// handler is the consuming side, built on so the validation +/// parameters can never drift from the minting side ( +/// sets RoleClaimType/NameClaimType to the minted keys, so [Authorize(Roles)] and +/// IsInRole resolve). +/// +/// +/// Fail closed: no header ⇒ NoResult (the cookie scheme is not consulted by this handler — the +/// policy scheme selects one or the other); a malformed, expired, tampered or wrong-issuer token ⇒ +/// Fail, never a fall-through to the cookie. A direct AddJwtBearer needs the +/// Microsoft.AspNetCore.Authentication.JwtBearer package, which the central package list does +/// not carry; this handler is ~40 lines over the already-referenced System.IdentityModel.Tokens.Jwt +/// and is a drop-in to replace once that package is pinned. +/// +public sealed class JwtBearerAuthenticationHandler : AuthenticationHandler +{ + private readonly JwtTokenService _jwt; + + /// Initializes a new instance of the class. + /// The scheme options monitor. + /// The logger factory. + /// The URL encoder. + /// The token service whose validation parameters mirror its minting side. + public JwtBearerAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder, + JwtTokenService jwt) + : base(options, logger, encoder) + { + _jwt = jwt; + } + + /// + protected override Task HandleAuthenticateAsync() + { + if (!OtOpcUaAuthSchemes.TryGetBearerToken(Request.Headers.Authorization, out var token)) + return Task.FromResult(AuthenticateResult.NoResult()); + + if (!_jwt.TryValidate(token, out var principal) || principal is null) + return Task.FromResult(AuthenticateResult.Fail("Invalid bearer token.")); + + return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(principal, Scheme.Name))); + } + + /// + protected override Task HandleChallengeAsync(AuthenticationProperties properties) + { + Response.StatusCode = StatusCodes.Status401Unauthorized; + Response.Headers.WWWAuthenticate = OtOpcUaAuthSchemes.Bearer; + return Task.CompletedTask; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs deleted file mode 100644 index 0e42b789..00000000 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System.Net.Http.Json; -using System.Security.Claims; -using Microsoft.AspNetCore.Components.Authorization; -using Microsoft.Extensions.Logging; - -namespace ZB.MOM.WW.OtOpcUa.Security.Blazor; - -/// -/// Blazor Server that snapshots the cookie-backed -/// principal supplied at circuit boot and polls /auth/ping every 60 seconds to detect -/// expiry. Mirrors ScadaLink's CentralUI implementation. -/// -public sealed class CookieAuthenticationStateProvider : AuthenticationStateProvider, IAsyncDisposable -{ - private static readonly TimeSpan PingInterval = TimeSpan.FromSeconds(60); - - private readonly HttpClient _http; - private readonly ILogger _logger; - private readonly CancellationTokenSource _cts = new(); - private ClaimsPrincipal _current; - private Task? _pingLoop; - - /// Initializes a new instance of the CookieAuthenticationStateProvider class. - /// The initial claims principal from circuit boot. - /// The HTTP client for authentication ping requests. - /// The logger for diagnostic messages. - public CookieAuthenticationStateProvider( - ClaimsPrincipal initial, - HttpClient http, - ILogger logger) - { - _current = initial; - _http = http; - _logger = logger; - } - - /// - public override Task GetAuthenticationStateAsync() - { - _pingLoop ??= Task.Run(() => PingLoopAsync(_cts.Token)); - return Task.FromResult(new AuthenticationState(_current)); - } - - private async Task PingLoopAsync(CancellationToken ct) - { - try - { - while (!ct.IsCancellationRequested) - { - await Task.Delay(PingInterval, ct).ConfigureAwait(false); - var resp = await _http.GetAsync("/auth/ping", ct).ConfigureAwait(false); - if (!resp.IsSuccessStatusCode && _current.Identity?.IsAuthenticated == true) - { - _logger.LogInformation("/auth/ping returned {Code}; notifying circuit", (int)resp.StatusCode); - _current = new ClaimsPrincipal(new ClaimsIdentity()); - NotifyAuthenticationStateChanged( - Task.FromResult(new AuthenticationState(_current))); - } - } - } - catch (OperationCanceledException) { /* expected on shutdown */ } - catch (Exception ex) - { - _logger.LogWarning(ex, "Auth ping loop terminated unexpectedly"); - } - } - - /// Cancels the background ping loop, awaits its completion, and disposes the cancellation token source. - /// A task that represents the asynchronous operation. - public async ValueTask DisposeAsync() - { - _cts.Cancel(); - if (_pingLoop is not null) - { - try { await _pingLoop.ConfigureAwait(false); } catch { /* swallow shutdown errors */ } - } - _cts.Dispose(); - } -} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs index 56ac6ef7..6b0876d7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs @@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using ZB.MOM.WW.Auth.AspNetCore; using ZB.MOM.WW.Auth.Abstractions.Roles; using ZB.MOM.WW.OtOpcUa.Security.Jwt; @@ -36,7 +37,14 @@ public static class AuthEndpoints // AddOtOpcUaAuth fallback policy that would otherwise 401 the request. app.MapPost("/auth/login", (Delegate)LoginAsync).AllowAnonymous().DisableAntiforgery(); app.MapGet("/auth/ping", (Delegate)Ping).AllowAnonymous(); - app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(); + // Mapped only when a signing key is configured (JwtOptions.IsConfigured): an unconfigured + // endpoint used to answer HTTP 500 from the JwtTokenService constructor on every call. + if (app.ServiceProvider.GetRequiredService>().Value.IsConfigured) + app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(); + else + app.ServiceProvider.GetService()? + .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") + .LogInformation("Security:Jwt:SigningKey is not configured — /auth/token is not mapped and bearer tokens are not accepted"); app.MapPost("/auth/logout", (Delegate)LogoutAsync).RequireAuthorization(); return app; } @@ -73,8 +81,13 @@ public static class AuthEndpoints { result = await ldap.AuthenticateAsync(username, password, ct); } - catch (Exception) + catch (Exception ex) when (ex is not OperationCanceledException) { + // R03:2.7 — a directory outage used to leave NOTHING in the server log from this path; the + // operator had to infer every 503 from the directory side. + http.RequestServices.GetService()? + .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") + .LogWarning(ex, "LDAP authentication failed for {User}; answering 503", username); return Results.StatusCode(StatusCodes.Status503ServiceUnavailable); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs index 07cbdda4..f523a033 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs @@ -15,4 +15,26 @@ public sealed class JwtOptions /// Default token expiry. Mirrors ScadaLink (15 min). public int ExpiryMinutes { get; set; } = 15; + + /// + /// Whether the JWT feature is on: a non-blank . When off, /auth/token + /// is not mapped and no bearer scheme is registered, so an admin node that never configured a key + /// keeps booting exactly as before; when on, the key is validated at host start + /// (ValidateOnStart) instead of throwing HTTP 500 on the first token request. + /// + public bool IsConfigured => !string.IsNullOrWhiteSpace(SigningKey); + + /// The validation rule shared by the startup validator and the tests: why the options are invalid, or null. + /// A failure message, or when valid. + public string? Validate() + { + if (!IsConfigured) return null; + var keyBytes = System.Text.Encoding.UTF8.GetByteCount(SigningKey); + if (keyBytes < MinSigningKeyBytes) + return $"{SectionName}:SigningKey must be at least {MinSigningKeyBytes} bytes (256 bits) UTF-8 for HMAC-SHA256; the configured key is {keyBytes} byte(s)."; + if (string.IsNullOrWhiteSpace(Issuer)) return $"{SectionName}:Issuer must not be blank."; + if (string.IsNullOrWhiteSpace(Audience)) return $"{SectionName}:Audience must not be blank."; + if (ExpiryMinutes <= 0) return $"{SectionName}:ExpiryMinutes must be > 0."; + return null; + } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs new file mode 100644 index 00000000..c3805267 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.Options; + +namespace ZB.MOM.WW.OtOpcUa.Security.Jwt; + +/// +/// Startup validator for (R03:2.4): when a signing key is configured it must +/// be usable, and the host must learn that at start (ValidateOnStart) rather than from an HTTP 500 +/// on the first /auth/token. An unconfigured section (no key) is valid — the feature is simply off. +/// +public sealed class JwtOptionsValidator : IValidateOptions +{ + /// + public ValidateOptionsResult Validate(string? name, JwtOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var failure = options.Validate(); + return failure is null ? ValidateOptionsResult.Success : ValidateOptionsResult.Fail(failure); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs index 6df24cad..d58fbcf3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs @@ -19,9 +19,12 @@ using ZB.MOM.WW.OtOpcUa.Security.Ldap; namespace ZB.MOM.WW.OtOpcUa.Security; /// -/// DI registration for OtOpcUa auth. Single Cookie scheme (the JWT lives inside the -/// cookie as its credential payload); no JwtBearer parallel scheme. Matches ScadaBridge -/// structurally — see docs/plans/2026-05-29-auth-alignment-design.md. +/// DI registration for OtOpcUa auth. The cookie scheme carries the browser session; when +/// Security:Jwt:SigningKey is configured a scheme +/// accepts the tokens /auth/token mints (R03:2.4), and the default scheme is a policy scheme +/// that forwards to whichever the request presents — so /api/* and every +/// RequireAuthorization endpoint take a browser session OR a machine token. Matches +/// ScadaBridge structurally — see docs/plans/2026-05-29-auth-alignment-design.md. /// public static class ServiceCollectionExtensions { @@ -36,7 +39,14 @@ public static class ServiceCollectionExtensions /// The same service collection, for chaining. public static IServiceCollection AddOtOpcUaAuth(this IServiceCollection services, IConfiguration configuration) { - services.AddOptions().Bind(configuration.GetSection(JwtOptions.SectionName)); + // JWT is opt-in by key: with no SigningKey the feature is off (no /auth/token, no bearer scheme) + // and an admin node boots as before; with one, the key is validated at host START (R03:2.4) — + // the JwtTokenService constructor used to throw on the first /auth/token call (HTTP 500). + services.AddOptions() + .Bind(configuration.GetSection(JwtOptions.SectionName)) + .ValidateOnStart(); + services.AddSingleton, JwtOptionsValidator>(); + var jwtConfigured = configuration.GetSection(JwtOptions.SectionName).Get()?.IsConfigured ?? false; services.AddOptions().Bind(configuration.GetSection(OtOpcUaCookieOptions.SectionName)); services.AddOptions().Bind(configuration.GetSection(LdapOptions.SectionName)); services.AddOptions().Bind(configuration.GetSection(AuthDisableLoginOptions.SectionName)); @@ -53,7 +63,10 @@ public static class ServiceCollectionExtensions // as the Actor field when constructing a canonical ZB.MOM.WW.Audit.AuditEvent. Registered // Scoped so it correctly follows the request scope used by Blazor Server and minimal-API // endpoints. - services.TryAddScoped(); + // AuthenticationStateAuditActorAccessor reads the Blazor circuit's principal first (no HttpContext + // exists on a circuit's threads after the first render) and falls back to the HTTP context for + // minimal-API scopes, where AuthenticationStateProvider is not registered (R03:2.5). + services.TryAddScoped(); // Singleton — OtOpcUaLdapAuthService is stateless (the shared-library directory client it // wraps opens/disposes an LdapConnection per call) and must be consumable by the Singleton @@ -78,7 +91,26 @@ public static class ServiceCollectionExtensions .GetSection(AuthDisableLoginOptions.SectionName) .GetValue(nameof(AuthDisableLoginOptions.DisableLogin)); - var authBuilder = services.AddAuthentication(CookieAuthenticationDefaults.AuthenticationScheme); + // The default scheme is the cookie-or-bearer selector; sign-in/out stay on the cookie scheme, which is + // the only one that can issue a session. With JWT off the selector always picks the cookie. + var authBuilder = services.AddAuthentication(o => + { + o.DefaultScheme = OtOpcUaAuthSchemes.CookieOrBearer; + o.DefaultSignInScheme = CookieAuthenticationDefaults.AuthenticationScheme; + o.DefaultSignOutScheme = CookieAuthenticationDefaults.AuthenticationScheme; + }); + authBuilder.AddPolicyScheme(OtOpcUaAuthSchemes.CookieOrBearer, "Cookie or Bearer", o => + { + o.ForwardDefaultSelector = ctx => + jwtConfigured && OtOpcUaAuthSchemes.HasBearerToken(ctx) + ? OtOpcUaAuthSchemes.Bearer + : CookieAuthenticationDefaults.AuthenticationScheme; + }); + if (jwtConfigured) + { + authBuilder.AddScheme( + OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { }); + } if (disableLogin) { // DEV/TEST ONLY: replace the cookie handler with an always-succeeding handler registered @@ -142,8 +174,8 @@ public static class ServiceCollectionExtensions services.AddAuthorization(o => { - o.FallbackPolicy = new AuthorizationPolicyBuilder( - CookieAuthenticationDefaults.AuthenticationScheme) + // Named on the selector, not the cookie scheme, so the fallback accepts a bearer token too. + o.FallbackPolicy = new AuthorizationPolicyBuilder(OtOpcUaAuthSchemes.CookieOrBearer) .RequireAuthenticatedUser() .Build(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs new file mode 100644 index 00000000..f4e6fd22 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs @@ -0,0 +1,68 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:4.1 — admin operations echoed ex.Message to the UI, so a SqlException's connection-string +/// fragment reached any ConfigEditor browser session. The reply is now a fixed sentence plus the request's +/// correlation id; the detail stays in the central log. +/// +public sealed class AdminOperationsActorServerFaultMessageTests : ControlPlaneActorTestBase +{ + private const string Secret = "Server=sql-prod-01;Password=hunter2"; + + [Fact] + public void StartDeployment_server_fault_reply_carries_the_correlation_id_and_not_the_exception_text() + { + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(new ThrowingDbFactory(), coordinator.Ref, Enumerable.Empty())); + var correlationId = CorrelationId.NewId(); + + actor.Tell(new StartDeployment("joe", correlationId)); + + var reply = ExpectMsg(PresenceBudget); + reply.Outcome.ShouldBe(StartDeploymentOutcome.Rejected); + reply.Message.ShouldNotBeNull(); + reply.Message.ShouldNotContain("hunter2"); + reply.Message.ShouldNotContain("sql-prod-01"); + reply.Message.ShouldContain(correlationId.ToString()); // the operator can find the log line + } + + [Fact] + public void TestDriverConnect_probe_throw_reply_carries_the_correlation_id_and_not_the_exception_text() + { + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(NewInMemoryDbFactory(), coordinator.Ref, new IDriverProbe[] { new ThrowingProbe() })); + var correlationId = Guid.NewGuid(); + + actor.Tell(new TestDriverConnect("Throwing", "{}", 5, correlationId)); + + var reply = ExpectMsg(PresenceBudget); + reply.Ok.ShouldBeFalse(); + reply.Message.ShouldNotBeNull(); + reply.Message.ShouldNotContain("hunter2"); + reply.Message.ShouldContain(correlationId.ToString()); + } + + private sealed class ThrowingDbFactory : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => + throw new InvalidOperationException("Cannot open database: " + Secret); + } + + private sealed class ThrowingProbe : IDriverProbe + { + public string DriverType => "Throwing"; + public Task ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct) => + throw new InvalidOperationException("probe blew up: " + Secret); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs new file mode 100644 index 00000000..e91fbda0 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration; + +/// +/// R03:2.3 — an unset or placeholder pepper must fail host start, not surface as "Invalid API key" +/// on the first activation. +/// +public sealed class ApiKeyOptionsValidatorTests +{ + private static ValidateOptionsResult Validate(ApiKeyOptions options) => + new ApiKeyOptionsValidator().Validate(Options.DefaultName, options); + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Blank_pepper_fails(string pepper) + { + var result = Validate(new ApiKeyOptions { Pepper = pepper }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain("Pepper is required"); + } + + [Fact] + public void Short_pepper_fails() + { + var result = Validate(new ApiKeyOptions { Pepper = "x" }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain($"at least {ApiKeyOptionsValidator.MinPepperLength}"); + } + + [Fact] + public void Blank_username_fails() + { + var result = Validate(new ApiKeyOptions { Pepper = new string('p', 32), Username = " " }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain("Username"); + } + + /// Positive control: the docker-dev rig's pepper and the defaults pass unchanged. + [Fact] + public void Configured_pepper_passes() + { + Validate(new ApiKeyOptions { Pepper = "docker-dev-apikey-pepper-not-for-production" }) + .Succeeded.ShouldBeTrue(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs new file mode 100644 index 00000000..5dc1f391 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs @@ -0,0 +1,80 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Components.Authorization; +using Microsoft.AspNetCore.Http; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Auth.AspNetCore; +using ZB.MOM.WW.OtOpcUa.Security.Audit; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests.Audit; + +/// +/// R03:2.5 — inside a Blazor circuit there is no HttpContext after the first render, so an +/// HTTP-only accessor stamps every audit row from an interactive page "system". The circuit's +/// is consulted first; the HTTP context is the fallback. +/// +public sealed class AuthenticationStateAuditActorAccessorTests +{ + [Fact] + public void Circuit_principal_wins_when_there_is_no_http_context() + { + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new FixedStateProvider(Authenticated(new Claim(ZbClaimTypes.Username, "alice")))); + + accessor.CurrentActor.ShouldBe("alice"); + } + + [Fact] + public void Http_principal_is_the_fallback_outside_a_circuit() + { + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "bob")) }; + var accessor = new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), authenticationState: null); + + accessor.CurrentActor.ShouldBe("bob"); + } + + [Fact] + public void Unauthenticated_circuit_falls_through_to_http_then_null() + { + var anonymousCircuit = new FixedStateProvider(new ClaimsPrincipal(new ClaimsIdentity())); + + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(null), anonymousCircuit) + .CurrentActor.ShouldBeNull(); + + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Name, "carol")) }; + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), anonymousCircuit) + .CurrentActor.ShouldBe("carol"); + } + + /// An incomplete state task is "no circuit identity", never a block. + [Fact] + public void Pending_authentication_state_is_not_awaited() + { + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new PendingStateProvider()); + + accessor.CurrentActor.ShouldBeNull(); + } + + private static ClaimsPrincipal Authenticated(params Claim[] claims) => + new(new ClaimsIdentity(claims, "TestScheme", ZbClaimTypes.Name, ZbClaimTypes.Role)); + + private sealed class HttpContextAccessorStub(HttpContext? context) : IHttpContextAccessor + { + public HttpContext? HttpContext { get; set; } = context; + } + + private sealed class FixedStateProvider(ClaimsPrincipal user) : AuthenticationStateProvider + { + public override Task GetAuthenticationStateAsync() => + Task.FromResult(new AuthenticationState(user)); + } + + private sealed class PendingStateProvider : AuthenticationStateProvider + { + public override Task GetAuthenticationStateAsync() => + new TaskCompletionSource().Task; + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs index 9cf8ab11..e5e3b620 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs @@ -11,6 +11,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Shouldly; using Xunit; using ZB.MOM.WW.Auth.AspNetCore; @@ -37,6 +38,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime private IHost _host = null!; private TestServer _server = null!; private readonly StubLdapGroupRoleMappingService _roleMappings = new(); + private readonly CapturingLoggerProvider _logs = new(); private static CancellationToken Ct => TestContext.Current.CancellationToken; @@ -70,6 +72,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime ["Security:Ldap:GroupToRole:ReadOnly"] = "Viewer", }).Build(); services.AddOtOpcUaAuth(configuration); + services.AddLogging(l => l.AddProvider(_logs)); services.AddSingleton(); // The login handler now resolves the DB role-map service via DI to merge // DB-backed grants on top of the appsettings baseline. Register the stub so @@ -96,6 +99,10 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime .ToArray(); return Results.Ok(claims); }).RequireAuthorization(); + // A role-gated machine endpoint: proves a bearer principal's roles resolve through + // IsInRole (the validation parameters pair RoleClaimType with the minted key). + e.MapGet("/api/viewer-only", () => Results.Ok("viewer")) + .RequireAuthorization(p => p.RequireRole("Viewer")); }); }); }) @@ -153,6 +160,66 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime new AuthEndpoints.LoginRequest("ldap-down", "anything"), Ct); response.StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + // R03:2.7 — the outage is named in the server log, not inferred from the directory side. + _logs.Entries.Any(e => + e.Level == LogLevel.Warning && e.Message.Contains("LDAP authentication failed") && e.Exception != null) + .ShouldBeTrue("a Warning naming the LDAP failure must be logged"); + } + + /// + /// R03:2.4 — a token from /auth/token authenticates a request carrying it as a bearer: + /// the fallback-protected root AND a role-gated endpoint (the bearer principal's "Role" claims + /// resolve through IsInRole). No cookie is sent. + /// + [Fact] + public async Task Bearer_token_authenticates_api_requests_without_a_cookie() + { + var token = await LoginAndIssueTokenAsync(); + var client = NewClient(); + + var root = new HttpRequestMessage(HttpMethod.Get, "/"); + root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + (await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + var gated = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + gated.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + var gatedResponse = await client.SendAsync(gated, Ct); + gatedResponse.StatusCode.ShouldBe(HttpStatusCode.OK); + (await gatedResponse.Content.ReadAsStringAsync(Ct)).ShouldContain("viewer"); + } + + /// A tampered or garbage bearer fails closed (401), it never falls through to anonymous. + [Theory] + [InlineData("not-a-jwt")] + [InlineData("tampered")] + public async Task Bad_bearer_token_is_rejected(string kind) + { + var token = kind == "tampered" + ? (await LoginAndIssueTokenAsync()) + "x" + : kind; + var client = NewClient(); + var req = new HttpRequestMessage(HttpMethod.Get, "/"); + req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + req.Headers.Add("X-Requested-With", "XMLHttpRequest"); + + var response = await client.SendAsync(req, Ct); + + response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized); + response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); + } + + private async Task LoginAndIssueTokenAsync() + { + var client = NewClient(); + var loginResponse = await client.PostAsJsonAsync("/auth/login", + new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + loginResponse.EnsureSuccessStatusCode(); + var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + AttachCookies(tokenReq, loginResponse); + var response = await client.SendAsync(tokenReq, Ct); + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadFromJsonAsync(Ct); + return payload.GetProperty("token").GetString()!; } /// Tests that ping anonymous returns 401. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs new file mode 100644 index 00000000..a79ab569 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs @@ -0,0 +1,25 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests; + +/// Collects every log entry written through the host's logger factory, for assertions on what was logged. +internal sealed class CapturingLoggerProvider : ILoggerProvider +{ + public ConcurrentQueue Entries { get; } = new(); + + public ILogger CreateLogger(string categoryName) => new Logger(categoryName, Entries); + + public void Dispose() { } + + internal sealed record Entry(string Category, LogLevel Level, string Message, Exception? Exception); + + private sealed class Logger(string category, ConcurrentQueue entries) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + entries.Enqueue(new Entry(category, logLevel, formatter(state, exception), exception)); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs new file mode 100644 index 00000000..96fc88ca --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs @@ -0,0 +1,127 @@ +using System.Net; +using System.Net.Http.Json; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Services; +using ZB.MOM.WW.OtOpcUa.Security.Auth; +using ZB.MOM.WW.OtOpcUa.Security.Endpoints; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; +using ZB.MOM.WW.OtOpcUa.Security.Ldap; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests; + +/// +/// R03:2.4 — JWT is opt-in by signing key. Unconfigured: /auth/token is not mapped (it used to +/// answer HTTP 500 from the service constructor) and no bearer scheme exists. Configured badly: the +/// host refuses to START rather than failing on the first token request. +/// +public sealed class JwtConfigurationGateTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Without_a_signing_key_auth_token_is_not_mapped_and_bearer_is_ignored() + { + using var host = await StartHostAsync(signingKey: null); + var server = host.GetTestServer(); + var client = server.CreateClient(); + + // The login still works — the cookie path is untouched. + var login = await client.PostAsJsonAsync("/auth/login", new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + login.StatusCode.ShouldBe(HttpStatusCode.NoContent); + + // /auth/token is simply absent (404), not a 500. + var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + foreach (var c in login.Headers.GetValues("Set-Cookie")) tokenReq.Headers.Add("Cookie", c.Split(';')[0]); + (await client.SendAsync(tokenReq, Ct)).StatusCode.ShouldBe(HttpStatusCode.NotFound); + + // No bearer scheme is registered, so a bearer header is ignored and the cookie session still wins. + var schemes = host.Services.GetRequiredService(); + (await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer)).ShouldBeNull(); + var withBogusBearer = new HttpRequestMessage(HttpMethod.Get, "/"); + foreach (var c in login.Headers.GetValues("Set-Cookie")) withBogusBearer.Headers.Add("Cookie", c.Split(';')[0]); + withBogusBearer.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "garbage"); + (await client.SendAsync(withBogusBearer, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + await host.StopAsync(Ct); + } + + /// Positive control for the gate: with a key, the bearer scheme IS registered. + [Fact] + public async Task With_a_signing_key_the_bearer_scheme_is_registered() + { + using var host = await StartHostAsync(signingKey: "test-signing-key-with-at-least-32-bytes-of-utf8-content"); + var schemes = host.Services.GetRequiredService(); + var bearer = await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer); + bearer.ShouldNotBeNull(); + bearer.HandlerType.ShouldBe(typeof(JwtBearerAuthenticationHandler)); + await host.StopAsync(Ct); + } + + /// A configured-but-short key fails at host start (ValidateOnStart), not on the first /auth/token. + [Fact] + public async Task A_short_signing_key_fails_host_start() + { + var ex = await Should.ThrowAsync(async () => + { + using var host = await StartHostAsync(signingKey: "too-short"); + }); + ex.Message.ShouldContain("SigningKey"); + } + + private static async Task StartHostAsync(string? signingKey) + { + var dbName = $"jwt-gate-{Guid.NewGuid():N}"; + var settings = new Dictionary(); + if (signingKey is not null) settings["Security:Jwt:SigningKey"] = signingKey; + var configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build(); + + var host = new HostBuilder() + .ConfigureWebHost(web => + { + web.UseTestServer(); + web.ConfigureServices(services => + { + services.AddDbContextFactory(opt => opt.UseInMemoryDatabase(dbName)); + services.AddDbContext(opt => opt.UseInMemoryDatabase(dbName)); + services.AddRouting(); + services.AddOtOpcUaAuth(configuration); + services.AddSingleton(); + services.AddSingleton(); + }); + web.Configure(app => + { + app.UseRouting(); + app.UseAuthentication(); + app.UseAuthorization(); + app.UseEndpoints(e => + { + e.MapOtOpcUaAuth(); + e.MapGet("/", () => Results.Ok("authenticated")).RequireAuthorization(); + }); + }); + }) + .Build(); + await host.StartAsync(Ct); + return host; + } + + private sealed class AlwaysAliceLdap : ILdapAuthService + { + public Task AuthenticateAsync(string username, string password, CancellationToken ct = default) => + Task.FromResult(new LdapAuthResult( + Success: password == "valid-password", DisplayName: "Alice", Username: username, + Groups: [], Roles: [], Error: null)); + } +} From 4d3fc9c8167b075fbaee1bfa0121ef6070efba1c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:27:49 -0400 Subject: [PATCH 09/27] fix(runtime): restart-safe alarm host, mux last-value replay, VT log rate limit, Calculation remediation (E4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R02:H1 ScriptedAlarmHostActor gains a factory Props (Func<(upstream, engine)>) so every incarnation builds and owns its own engine; a supervision restart no longer re-creates the actor around a disposed engine. The legacy instance-taking Props stays source-compatible and is now restart-safe in the weaker sense (PreRestart marks the stop as a restart, PostStop then detaches instead of disposing). DriverHostActor still spawns through the legacy overload (not owned by this lane — F3). - R02:H3 OnAlarmCommand runs every engine op under a 10 s token linked to the actor's lifetime (DefaultCommandTimeout / Props commandTimeout) and logs a Warning on timeout instead of suspending the mailbox on a stalled store. - R02:L3 DependencyMuxTagUpstreamSource.RetainOnly(loaded dep-ref union), called after every successful load. - R02:P6 VirtualTagActor's per-failure ScriptLogEntry is rate-limited per tag (ScriptLogRateLimiter, 20/min + one summary); the Bad publish and the metric are untouched. - R07:H6 DependencyMuxActor retains the last (value, status, ts) per ever-registered ref (through unregisters) and replays them in one batch to a RegisterInterest(ReplayRetained: true) — VirtualTagActor, the Calculation mux adapter and the alarm host opt in; the historization recorder does not (a replay there would record a stale sample per deploy). CalculationDriver records inputs regardless of _running so a replay landing before InitializeAsync still satisfies the change-gate. - R07:P4 CalculationEvaluator negative-caches compile failures by source hash (one Roslyn attempt + one Warning per source until redeploy); the driver's script-log entry fires on the Good→Bad transition or a change of reason, matching the once-per-transition Bad publish. - R07:H8 array results dedup structurally. - R07:L2 ShutdownAsync drops the compiled scripts (collectible ALCs). - R07:U5 DraftValidator rule CalculationNoEffectiveTrigger: a tag with no timer whose change trigger is off or whose script has no literal ctx.GetTag read can never evaluate. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Validation/DraftValidator.cs | 48 +++++ .../CalculationDriver.cs | 48 ++++- .../CalculationEvaluator.cs | 39 +++- .../DependencyMuxTagUpstreamSource.cs | 28 ++- .../ScriptedAlarms/ScriptedAlarmHostActor.cs | 161 ++++++++++++++--- .../DependencyConsumerMuxAdapter.cs | 5 +- .../VirtualTags/DependencyMuxActor.cs | 68 ++++--- .../VirtualTags/VirtualTagActor.cs | 36 +++- .../DraftValidatorCalculationTests.cs | 56 +++++- .../CalculationDriverRemediationTests.cs | 166 ++++++++++++++++++ .../CalculationDriverTests.cs | 6 +- .../DependencyMuxTagUpstreamSourceTests.cs | 22 +++ .../ScriptedAlarmHostActorRestartTests.cs | 145 +++++++++++++++ .../VirtualTags/DependencyMuxReplayTests.cs | 101 +++++++++++ ...VirtualTagActorFailureLogRateLimitTests.cs | 55 ++++++ 15 files changed, 921 insertions(+), 63 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs index 04e08fe9..3b8d0bb1 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs @@ -452,6 +452,54 @@ public static class DraftValidator "Calculation tags form a dependency cycle (members: " + string.Join(", ", cycle) + "); a calc→calc cycle is a live oscillation loop and must be broken before deploy", cycle.Count > 0 ? cycle[0] : null)); + + // Rule 3 — effective-trigger gate (R07:U5). A calc tag evaluates on a declared dependency + // changing (change-trigger: the literal ctx.GetTag paths) and/or on a timer. A change-triggered + // tag whose script has NO literal reads (`return ctx.Now.Hour;`, a dynamic-path read) and no timer + // never evaluates at all: it sits BadWaitingForInitialData forever while the driver reports + // Healthy, and nothing warned. Same for a tag that is neither change- nor timer-triggered. + foreach (var (t, _, scriptId) in calcTags) + { + var (changeTriggered, timerIntervalMs) = ParseTriggers(t.TagConfig); + if (timerIntervalMs is > 0) continue; + var source = scriptId is not null && scriptSourceById.TryGetValue(scriptId, out var s) ? s : null; + var depCount = EquipmentScriptPaths.ExtractDependencyRefs(source).Count; + if (changeTriggered && depCount > 0) continue; + errors.Add(new("CalculationNoEffectiveTrigger", + changeTriggered + ? $"Calculation tag '{t.TagId}' is change-triggered but its script has no literal ctx.GetTag read, " + + "and it has no timerIntervalMs — nothing would ever evaluate it. Add a dependency read or a timer." + : $"Calculation tag '{t.TagId}' has changeTriggered=false and no timerIntervalMs — nothing would ever evaluate it. " + + "Enable the change trigger or add a timer.", + t.TagId)); + } + } + + /// Read the trigger fields from a calc tag's TagConfig: changeTriggered + /// (default true, the driver's own default) and timerIntervalMs (null when absent or not a + /// number). Never throws — malformed JSON yields the defaults. + private static (bool ChangeTriggered, int? TimerIntervalMs) ParseTriggers(string? tagConfig) + { + if (string.IsNullOrWhiteSpace(tagConfig)) return (true, null); + try + { + using var doc = System.Text.Json.JsonDocument.Parse(tagConfig); + var root = doc.RootElement; + if (root.ValueKind != System.Text.Json.JsonValueKind.Object) return (true, null); + var changeTriggered = true; + if (root.TryGetProperty("changeTriggered", out var ct) + && ct.ValueKind is System.Text.Json.JsonValueKind.False or System.Text.Json.JsonValueKind.True) + changeTriggered = ct.GetBoolean(); + int? timer = null; + if (root.TryGetProperty("timerIntervalMs", out var tm) + && tm.ValueKind == System.Text.Json.JsonValueKind.Number && tm.TryGetInt32(out var ms)) + timer = ms; + return (changeTriggered, timer); + } + catch (System.Text.Json.JsonException) + { + return (true, null); + } } /// Extract the scriptId string from a calc tag's schemaless TagConfig JSON. Never diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs index dc0d8ba1..905fff13 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs @@ -338,6 +338,11 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba StopTimers(); // The pump exits at its next iteration; an evaluation already in flight is bounded by RunTimeout. lock (_lock) _pending.Clear(); + // R07:L2 — nothing in the runtime disposes a driver, and every config change is a respawn, so + // the compiled scripts' collectible ALCs survived each generation until GC got round to the whole + // driver graph. Drop them here (idempotent; a later InitializeAsync recompiles on first use). An + // in-flight run that races this is already handled by the evaluator's retry-once path. + _evaluator.ClearCompiledScripts(); return Task.CompletedTask; } @@ -363,11 +368,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba /// public void OnDependencyValue(string rawPath, object? value, uint statusCode, DateTime timestampUtc) { - if (!_running) return; lock (_lock) { + // Recorded REGARDLESS of _running (R07:H6): the mux adapter registers interest at construction, + // so inputs — the mux's retained-value replay in particular — can land before InitializeAsync. + // Dropping them left every calc tag waiting for each input to change again; keeping them means + // the first trigger after init evaluates from current inputs. Only the EVALUATION is gated. _depValues[rawPath] = new DataValueSnapshot(value, statusCode, timestampUtc, timestampUtc); _arrivedDeps.Add(rawPath); + if (!_running) return; if (!_changeDependents.TryGetValue(rawPath, out var dependents) || dependents.Count == 0) return; // QUEUE each change-triggered dependent — never evaluate here. This @@ -600,7 +609,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token) .ConfigureAwait(false); - string? failureReason = null; + string? failureToLog = null; (object? Value, uint StatusCode)? toPublish; lock (_lock) { @@ -610,16 +619,24 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba if (!result.Success) { - failureReason = result.Reason ?? "evaluator failure"; + var reason = result.Reason ?? "evaluator failure"; toPublish = MarkEvaluationFailed(st, pending.TimestampUtc); + // The script-log entry fires on the Good→Bad TRANSITION, or when the failure changes + // (R07:P4) — not on every trigger of a tag that is already Bad for the same reason, which + // flooded the script log at the input's cadence. The Bad publish was already once per + // transition; the log now matches it. + if (toPublish is not null || !string.Equals(st.LastFailureReason, reason, StringComparison.Ordinal)) + failureToLog = reason; + st.LastFailureReason = reason; } else { + st.LastFailureReason = null; toPublish = MarkEvaluationSucceeded(st, result.Value, pending.WorstInputStatus, pending.TimestampUtc); } } - if (failureReason is not null) PublishScriptLog(def, failureReason); + if (failureToLog is not null) PublishScriptLog(def, failureToLog); if (toPublish is { } p) Publish(def.RawPath, p.Value, p.StatusCode, pending.TimestampUtc); } @@ -638,7 +655,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba private (object? Value, uint StatusCode)? MarkEvaluationSucceeded( TagState st, object? value, uint statusCode, DateTime timestampUtc) { - if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && Equals(st.LastValue, value)) + if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && ValuesEqual(st.LastValue, value)) return null; st.HasValue = true; @@ -650,6 +667,17 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba return (value, statusCode); } + /// Value equality for the dedup: structural for arrays (R07:H8 — object.Equals is reference + /// equality for a double[], so an array-returning timer script republished every tick through + /// the dual-namespace fan-out and the historian), ordinary + /// otherwise. + private static bool ValuesEqual(object? a, object? b) + { + if (a is Array x && b is Array y) + return System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(x, y); + return Equals(a, b); + } + /// Record that this tag's INPUTS are Bad, so no evaluation ran. Publishes the /// worst input's status — carrying the last known value — once per change of that status: a /// repeat of the same Bad is silent, while BadNoCommunication arriving over an earlier @@ -727,6 +755,12 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba /// Test/diagnostic accessor: number of timer ticks skipped for in-flight groups. internal long TimerSkippedTickCount => _timers?.SkippedTickCount ?? 0; + /// Test/diagnostic accessor: sources currently remembered as failing to compile. + internal int CompileFailureCountForTest => _evaluator.CompileFailureCount; + + /// Test/diagnostic accessor: compiled scripts currently held by the evaluator's cache. + internal int CompiledScriptCountForTest => _evaluator.CompiledScriptCount; + /// Test/diagnostic accessor: when no change-triggered evaluation is /// queued or in flight. A test that used to assert straight after — /// which evaluated inline — waits on this instead, so the assertion still runs against a settled @@ -770,6 +804,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba /// now carries the failing INPUT's sub-code rather than a fixed BadInternalError. public uint LastPublishedStatus; public DateTime LastTimestampUtc; + + /// The reason of the last failed evaluation (null after a success / inputs-Bad hold) — the + /// gate that keeps the script-log entry to the transition and to a CHANGE of reason. + public string? LastFailureReason; } private sealed record CalculationSubscriptionHandle(string DiagnosticId) : ISubscriptionHandle; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs index 4296b993..ba4931c7 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using Microsoft.Extensions.Logging; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; @@ -58,6 +59,13 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO private readonly SerilogLogger _scriptRoot; private readonly TimeSpan _runTimeout; private readonly ScriptRunGuard _guard; + + // Negative compile cache (R07:P4), keyed by source hash. A compile error is a property of the SOURCE, + // which never changes between triggers (it is fixed in the deploy artifact) — yet every trigger of a + // broken tag used to re-run a full Roslyn compile under the driver-wide evaluation gate, starving the + // healthy tags behind it, and log the same Warning each time. The failure is now remembered until the + // next ClearCompiledScripts (a redeploy), and the host Warning fires once per source. + private readonly ConcurrentDictionary _compileFailures = new(StringComparer.Ordinal); private bool _disposed; /// Initializes a new . @@ -180,6 +188,11 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO calcTagId, path), logger: scriptLog); + // A source that already failed to compile fails the same way without touching Roslyn (R07:P4). + var sourceKey = HashSource(scriptSource); + if (_compileFailures.TryGetValue(sourceKey, out var knownFailure)) + return VirtualTagEvalResult.Failure(knownFailure); + // Fetch + run inside a bounded loop so an ObjectDisposedException caused by a concurrent // ClearCompiledScripts (which disposed the evaluator between GetOrCompile and the run's // disposed guard) re-fetches and retries exactly once. The loop runs at most twice. @@ -192,13 +205,11 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO } catch (CompilationErrorException ex) { - _logger.LogWarning(ex, "Calculation {Id}: Roslyn compile failed", calcTagId); - return VirtualTagEvalResult.Failure($"compile error: {ex.Message}"); + return RememberCompileFailure(sourceKey, calcTagId, $"compile error: {ex.Message}", ex, "Roslyn compile failed"); } catch (ScriptSandboxViolationException ex) { - _logger.LogWarning(ex, "Calculation {Id}: sandbox violation", calcTagId); - return VirtualTagEvalResult.Failure($"sandbox violation: {ex.Message}"); + return RememberCompileFailure(sourceKey, calcTagId, $"sandbox violation: {ex.Message}", ex, "sandbox violation"); } catch (ObjectDisposedException) { @@ -258,10 +269,29 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO } } + /// Record a compile failure for and log it ONCE — the next trigger + /// of the same source returns the remembered reason without a Roslyn round-trip or a second Warning. + private VirtualTagEvalResult RememberCompileFailure(string sourceKey, string calcTagId, string reason, Exception ex, string what) + { + if (_compileFailures.TryAdd(sourceKey, reason)) + _logger.LogWarning(ex, "Calculation {Id}: {What} (remembered until the next redeploy)", calcTagId, what); + return VirtualTagEvalResult.Failure(reason); + } + + private static string HashSource(string source) + => Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(source))); + + /// Test/diagnostic: the number of sources currently remembered as failing to compile. + public int CompileFailureCount => _compileFailures.Count; + + /// Test/diagnostic: the number of compiled scripts currently cached. + public int CompiledScriptCount => _cache.Count; + /// public void ClearCompiledScripts() { _cache.Clear(); + _compileFailures.Clear(); // The compile-cache clear is this evaluator's deploy boundary — a redeploy lifts every quarantine // (an edited source already lifts its own through the guard's source identity). _guard.ResetAll(); @@ -279,5 +309,6 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO if (_disposed) return; _disposed = true; _cache.Dispose(); + _compileFailures.Clear(); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs index 9c7c741f..5466ab49 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs @@ -42,8 +42,10 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource // publish. private const uint StatusNeverPushed = OpcUaStatus.BadWaitingForInitialData; - // Intentionally never pruned: cold-start semantics depend on retained last values so - // ReadTag can answer synchronously for any path that has ever been pushed. + // Retained last values: ReadTag answers synchronously for any path that has been pushed. Pruned + // ONLY through RetainOnly — the host calls it with the loaded dependency-ref union after every + // (re)load, so a renamed/removed raw path (a RawTree rename is a surgical remove+add) stops + // costing a boxed value + two DateTimes for the process lifetime (R02:L3). private readonly ConcurrentDictionary _cache = new(StringComparer.Ordinal); private readonly ConcurrentDictionary>> _observers @@ -91,6 +93,28 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource } } + /// + /// Drop every cached snapshot whose path is not in . Observers are + /// untouched. Called by the host after each successful engine (re)load with the union of the loaded + /// alarms' dependency refs — exactly the set the mux will push from now on — so paths no alarm + /// references any more stop being retained. + /// + /// The paths to keep. + /// The number of cached snapshots dropped. + public int RetainOnly(IReadOnlySet paths) + { + ArgumentNullException.ThrowIfNull(paths); + var dropped = 0; + foreach (var key in _cache.Keys) + { + if (!paths.Contains(key) && _cache.TryRemove(key, out _)) dropped++; + } + return dropped; + } + + /// Test/diagnostic: the number of paths currently retained. + public int RetainedPathCount => _cache.Count; + /// public DataValueSnapshot ReadTag(string path) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs index ffa5dc4f..210a84ca 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs @@ -23,10 +23,16 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; /// cluster alerts DistributedPubSub topic. /// /// -/// The host is the engine's lifecycle owner: the caller (the driver-role startup) builds -/// the , constructs a -/// around that same upstream, and passes both here. The host disposes the engine in -/// . +/// The host is the engine's lifecycle owner. The restart-safe shape (R02:H1) is the +/// factory : +/// every incarnation builds its own + a +/// around it, and disposes them in . +/// Akka's default restart runs PostStop then re-creates the actor from the SAME Props, so the +/// legacy instance-taking Props — a fixed engine captured in the Props — restarted around a +/// disposed engine and every later LoadAsync threw: the node's scripted alarms +/// silently died. The legacy overload is kept source-compatible but is restart-safe only in the +/// weaker sense that a restart now DETACHES rather than disposes its engine (see +/// ); production should spawn through the factory. /// /// /// @@ -99,6 +105,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor /// Warning on every clean shutdown. private sealed record AlarmsLoadCanceled; + /// Default bound on one inbound alarm command's engine op (): 10 s. + public static readonly TimeSpan DefaultCommandTimeout = TimeSpan.FromSeconds(10); + private readonly IActorRef _publishActor; private readonly IActorRef? _mux; private readonly DependencyMuxTagUpstreamSource _upstream; @@ -107,6 +116,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor private readonly ILoggingAdapter _log = Context.GetLogger(); private readonly CancellationTokenSource _cts = new(); private readonly EventHandler _onEngineEvent; + private readonly TimeSpan _commandTimeout; + + /// True when THIS incarnation built its engine (factory Props) and therefore disposes it on + /// every stop, restart included. False for the legacy instance-taking Props: the engine outlives the + /// incarnation, so a restart only detaches from it and a final stop disposes it. + private readonly bool _ownsEngine; + + /// Set by so the it triggers can tell a restart + /// (the next incarnation reuses a legacy engine) from a genuine stop. + private bool _restarting; /// Cached local from the latest /// snapshot (null = unknown until the first snapshot arrives, or no wired). The @@ -152,6 +171,8 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor /// Optional node-local live-telemetry hub (Phase 5) each cluster-wide /// alerts transition is also emitted into; null (tests) skips the emit. /// The used to instantiate the actor. + /// Legacy, instance-taking shape — see the class remarks: a supervision restart re-creates the + /// actor around these SAME instances, so prefer the factory overload in production. public static Props Props( IActorRef publishActor, IActorRef? mux, @@ -162,6 +183,60 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor ITelemetryLocalHub? telemetryHub = null) => Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider, telemetryHub)); + /// + /// Restart-safe factory Props (R02:H1): each incarnation of the actor calls + /// in its constructor and owns what it returns — the engine must + /// be constructed around the returned upstream — so a supervision restart gets a fresh, + /// undisposed engine and the next loads cleanly. + /// + /// The OPC UA publish actor emissions are bridged to. + /// Optional dependency multiplexer the host registers interest with. + /// Builds a fresh (upstream, engine) pair; called once per incarnation. + /// The local cluster node id for the Primary alerts-emit gate; null ⇒ default-emit. + /// Test seam for the alerts-emit gate's driver-member count. + /// Optional node-local live-telemetry hub. + /// Bound on one inbound alarm command's engine op; null ⇒ . + /// The used to instantiate the actor. + public static Props Props( + IActorRef publishActor, + IActorRef? mux, + Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory, + NodeId? localNode = null, + Func? driverMemberCountProvider = null, + ITelemetryLocalHub? telemetryHub = null, + TimeSpan? commandTimeout = null) => + Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, engineFactory, localNode, driverMemberCountProvider, telemetryHub, commandTimeout)); + + /// Factory-built incarnation: owns (and on every stop disposes) the engine it builds. + /// The OPC UA publish actor emissions are bridged to. + /// Optional dependency multiplexer the host registers interest with. + /// Builds this incarnation's (upstream, engine) pair. + /// The local cluster node id for the Primary alerts-emit gate. + /// Test seam for the alerts-emit gate's driver-member count. + /// Optional node-local live-telemetry hub. + /// Bound on one inbound alarm command's engine op. + public ScriptedAlarmHostActor( + IActorRef publishActor, + IActorRef? mux, + Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory, + NodeId? localNode = null, + Func? driverMemberCountProvider = null, + ITelemetryLocalHub? telemetryHub = null, + TimeSpan? commandTimeout = null) + : this(publishActor, mux, Build(engineFactory), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: true, commandTimeout) + { + } + + private static (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) Build( + Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory) + { + ArgumentNullException.ThrowIfNull(engineFactory); + var pair = engineFactory(); + ArgumentNullException.ThrowIfNull(pair.Upstream, "engineFactory().Upstream"); + ArgumentNullException.ThrowIfNull(pair.Engine, "engineFactory().Engine"); + return pair; + } + /// Initializes a new instance of the class. /// The OPC UA publish actor emissions are bridged to. /// Optional dependency multiplexer the host registers dependency interest with. @@ -183,14 +258,29 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor NodeId? localNode = null, Func? driverMemberCountProvider = null, ITelemetryLocalHub? telemetryHub = null) + : this(publishActor, mux, (upstream, engine), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: false, commandTimeout: null) + { + } + + private ScriptedAlarmHostActor( + IActorRef publishActor, + IActorRef? mux, + (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) pair, + NodeId? localNode, + Func? driverMemberCountProvider, + ITelemetryLocalHub? telemetryHub, + bool ownsEngine, + TimeSpan? commandTimeout) { ArgumentNullException.ThrowIfNull(publishActor); - ArgumentNullException.ThrowIfNull(upstream); - ArgumentNullException.ThrowIfNull(engine); + ArgumentNullException.ThrowIfNull(pair.Upstream); + ArgumentNullException.ThrowIfNull(pair.Engine); _publishActor = publishActor; _mux = mux; - _upstream = upstream; - _engine = engine; + _upstream = pair.Upstream; + _engine = pair.Engine; + _ownsEngine = ownsEngine; + _commandTimeout = commandTimeout is { } t && t > TimeSpan.Zero ? t : DefaultCommandTimeout; _localNode = localNode; _driverMemberCountProvider = driverMemberCountProvider; _telemetryHub = telemetryHub; @@ -294,8 +384,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor // subscriptions exist, so any DependencyValueChanged the mux delivers will be observed by the // engine. RegisterInterest replaces a subscriber's prior interest set (see DependencyMuxActor), // so a re-Apply with a fresh union simply supersedes the old one — no explicit unregister needed. - _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self)); - _log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs", msg.DepRefs.Count); + // ReplayRetained: the mux replays the last value it saw for each ref, so a fresh engine (a + // supervision restart builds one around an EMPTY upstream) converges immediately instead of + // waiting for every input to change again. + _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self, ReplayRetained: true)); + // Prune the upstream's retained snapshots to the loaded set (R02:L3): a path no alarm references + // any more will never be pushed again, so its last value is dead weight for the process lifetime. + var dropped = _upstream.RetainOnly(new HashSet(msg.DepRefs, StringComparer.Ordinal)); + _log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs (pruned {Dropped} retained path(s))", + msg.DepRefs.Count, dropped); ReassertConditionNodes(); } @@ -591,18 +688,25 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor return; } + // Bounded (R02:H3): every op ends in IAlarmStateStore.SaveAsync (LocalDb/SQLite), and this is a + // ReceiveAsync — the mailbox is suspended until the op returns, so an unbounded await on a hung + // file lock stalled dependency-value ingestion for the duration. Linked to the actor's lifetime + // token so a stop still cancels promptly. + using var cts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token); + cts.CancelAfter(_commandTimeout); + var ct = cts.Token; try { switch (cmd.Operation) { case "Acknowledge": - await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None); + await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct); break; case "Confirm": - await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None); + await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct); break; case "OneShotShelve": - await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, ct); break; case "TimedShelve": // A timed shelve needs the absolute unshelve instant. T18 derives it from the OPC UA @@ -614,16 +718,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor cmd.AlarmId); return; } - await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, CancellationToken.None); + await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, ct); break; case "Unshelve": - await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, ct); break; case "Enable": - await _engine.EnableAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.EnableAsync(cmd.AlarmId, cmd.User, ct); break; case "Disable": - await _engine.DisableAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.DisableAsync(cmd.AlarmId, cmd.User, ct); break; case "AddComment": if (string.IsNullOrWhiteSpace(cmd.Comment)) @@ -631,7 +735,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor _log.Warning("ScriptedAlarmHost: rejecting AddComment for {AlarmId} — empty comment text", cmd.AlarmId); return; } - await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None); + await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct); break; default: _log.Warning("ScriptedAlarmHost: ignoring AlarmCommand with unknown operation {Op} for {AlarmId}", @@ -639,6 +743,11 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor break; } } + catch (OperationCanceledException) when (!_cts.IsCancellationRequested) + { + _log.Warning("ScriptedAlarmHost: engine op {Op} for alarm {AlarmId} timed out after {Timeout} — the condition-state store is stalled", + cmd.Operation, cmd.AlarmId, _commandTimeout); + } catch (Exception ex) { // A failing engine op must not crash the actor — mirror OnLoadFailed's log-and-stay-inert style. @@ -672,16 +781,28 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor base.PreStart(); } + /// + /// Akka's default PreRestart stops the children and calls . The flag + /// lets that keep a legacy (Props-captured) engine alive for the next incarnation + /// instead of disposing the instance it is about to be re-created around. + protected override void PreRestart(Exception reason, object message) + { + _restarting = true; + base.PreRestart(reason, message); + } + /// protected override void PostStop() { // Unregister mux interest first so no further DependencyValueChanged arrives while we tear down. _mux?.Tell(new DependencyMuxActor.UnregisterInterest(Self)); // Cancel any in-flight LoadAsync, detach the OnEvent handler (so a late engine emission can't - // Tell a stopping actor), then dispose the engine (which drains its background work + clears). + // Tell a stopping actor), then dispose the engine (which drains its background work + clears) — + // unless this is a RESTART around a legacy Props-captured engine, which the next incarnation will + // re-attach to (R02:H1). _cts.Cancel(); _engine.OnEvent -= _onEngineEvent; - _engine.Dispose(); + if (_ownsEngine || !_restarting) _engine.Dispose(); _cts.Dispose(); base.PostStop(); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs index 7011492d..b8dc818f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs @@ -62,7 +62,10 @@ public sealed class DependencyConsumerMuxAdapter : ReceiveActor var refs = _consumer.DependencyRefs.ToList(); // A RegisterInterest replaces the prior interest set for this subscriber (mux is idempotent on // re-register), so re-registering after an apply cleanly adopts an added/removed ref. - _mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self)); + // ReplayRetained (R07:H6): a driver Restart is a NEW consumer behind a new adapter; the mux replays + // the last value of every ref it retained so the calc tags compute from current inputs at once + // instead of sitting BadWaitingForInitialData until each input happens to change. + _mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self, ReplayRetained: true)); _log.Debug("DependencyConsumerMuxAdapter: registered {Count} dependency ref(s)", refs.Count); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs index fe57da4e..3af36d6a 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs @@ -26,6 +26,20 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; /// receive as well, for direct senders and tests. /// The mux is deliberately not a DPS subscriber — virtual-tag evaluation is local to each node and /// would over-emit if it spanned the cluster. +/// +/// +/// Last-value retention + replay (R07:H6). For every ref that has EVER been registered the +/// mux retains the last (value, status, timestamp) it forwarded — through unregisters too, +/// because the case that matters is a subscriber that went away and came back: a Calculation +/// driver Restart is a new driver instance + a new mux adapter, and before this every calc tag sat +/// BadWaitingForInitialData until each of its inputs happened to change again (a +/// scripted-alarm host restart has the same shape). A with +/// receives one batch of the retained values for its +/// refs, in the same shape a driver burst would arrive in. Opt-in per registration: the +/// continuous-historization recorder re-registers on every deploy and a replay there would +/// record a stale sample per tag per deploy. Memory is bounded by the number of refs scripts have +/// ever named on this node, not by the address space. +/// /// public sealed class DependencyMuxActor : ReceiveActor { @@ -33,14 +47,23 @@ public sealed class DependencyMuxActor : ReceiveActor /// Register a subscriber's interest in a set of tag refs. Idempotent on re-register — /// the second call replaces the prior interest set for that subscriber. - public sealed record RegisterInterest(IReadOnlyList TagRefs, IActorRef Subscriber); + /// The refs the subscriber wants. + /// The subscriber. + /// When true, the mux immediately sends the subscriber one + /// holding the last retained value of every ref + /// in it has one for (nothing when it has none). + public sealed record RegisterInterest(IReadOnlyList TagRefs, IActorRef Subscriber, bool ReplayRetained = false); /// Unregister every interest held by . Sent on PostStop by /// the subscriber, or by Terminated handling when the mux watches. public sealed record UnregisterInterest(IActorRef Subscriber); + // A ref stays in _byRef once registered — with an EMPTY set after its last subscriber leaves — so + // the hot path's single lookup per item also decides retention. _byRef.Count == 0 keeps meaning + // "nothing was ever registered", which the batch path early-outs on. private readonly Dictionary> _byRef = new(StringComparer.Ordinal); private readonly Dictionary> _bySubscriber = new(); + private readonly Dictionary _last = new(StringComparer.Ordinal); private readonly ILoggingAdapter _log = Context.GetLogger(); /// Creates props for the DependencyMuxActor. @@ -64,16 +87,13 @@ public sealed class DependencyMuxActor : ReceiveActor { foreach (var r in priorRefs) { - if (_byRef.TryGetValue(r, out var set)) - { - set.Remove(msg.Subscriber); - if (set.Count == 0) _byRef.Remove(r); - } + if (_byRef.TryGetValue(r, out var set)) set.Remove(msg.Subscriber); } } var refs = new HashSet(msg.TagRefs, StringComparer.Ordinal); _bySubscriber[msg.Subscriber] = refs; + List? replay = null; foreach (var r in refs) { if (!_byRef.TryGetValue(r, out var set)) @@ -82,9 +102,19 @@ public sealed class DependencyMuxActor : ReceiveActor _byRef[r] = set; } set.Add(msg.Subscriber); + if (msg.ReplayRetained && _last.TryGetValue(r, out var retained)) + (replay ??= []).Add(retained); } Context.Watch(msg.Subscriber); _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs", msg.Subscriber, refs.Count); + + // One batch, after the registration is complete, in exactly the shape a driver burst arrives in — + // so the subscriber's hot path handles it with no special case. + if (replay is not null) + { + msg.Subscriber.Tell(new VirtualTagActor.DependencyValueBatch(replay)); + _log.Debug("DependencyMux: replayed {Count} retained value(s) to {Sub}", replay.Count, msg.Subscriber); + } } private void RemoveSubscriber(IActorRef subscriber) @@ -92,11 +122,9 @@ public sealed class DependencyMuxActor : ReceiveActor if (!_bySubscriber.TryGetValue(subscriber, out var refs)) return; foreach (var r in refs) { - if (_byRef.TryGetValue(r, out var set)) - { - set.Remove(subscriber); - if (set.Count == 0) _byRef.Remove(r); - } + // The ref's entry stays (possibly empty) so its last value keeps being retained for a + // subscriber that comes back — see the class remarks. + if (_byRef.TryGetValue(r, out var set)) set.Remove(subscriber); } _bySubscriber.Remove(subscriber); Context.Unwatch(subscriber); @@ -105,13 +133,14 @@ public sealed class DependencyMuxActor : ReceiveActor private void OnAttributeValuePublished(DriverInstanceActor.AttributeValuePublished msg) { - if (!_byRef.TryGetValue(msg.FullReference, out var subscribers) || subscribers.Count == 0) + if (!_byRef.TryGetValue(msg.FullReference, out var subscribers)) { - // No virtual tag cares about this ref — drop. Common in normal operation; the address + // No virtual tag ever cared about this ref — drop. Common in normal operation; the address // space carries thousands of tags and only a fraction feed virtual-tag expressions. return; } var dep = new VirtualTagActor.DependencyValueChanged(msg.FullReference, msg.Value, msg.TimestampUtc, msg.StatusCode); + _last[msg.FullReference] = dep; foreach (var sub in subscribers) { // A single-item batch, so the mux emits exactly ONE message shape and every subscriber needs @@ -122,15 +151,6 @@ public sealed class DependencyMuxActor : ReceiveActor } } - /// Filters one batched driver production burst per item and delivers each subscriber's matched - /// slice as ONE . Items nobody registered for are - /// dropped (the common case: only a fraction of an address space feeds virtual-tag expressions), so a - /// burst of N values costs one inbound message plus one outbound message per genuinely interested - /// SUBSCRIBER — not per (item, subscriber) pair as it did when this fanned out per value. That is what - /// lets evaluate its script once per burst instead of once per input. - /// The registered set is NOT just virtual-tag inputs: ContinuousHistorizationRecorder - /// registers interest in every historized ref, so on a historizing node can be - /// large and a sizeable fraction of each burst resolves. private void OnAttributeValueBatchPublished(DriverInstanceActor.AttributeValueBatchPublished msg) { // Nobody has registered anything — skip the whole burst rather than probing an empty map per item. @@ -146,8 +166,10 @@ public sealed class DependencyMuxActor : ReceiveActor for (var i = 0; i < count; i++) { var item = msg.Items[i]; - if (!_byRef.TryGetValue(item.FullReference, out var subscribers) || subscribers.Count == 0) continue; + if (!_byRef.TryGetValue(item.FullReference, out var subscribers)) continue; var dep = new VirtualTagActor.DependencyValueChanged(item.FullReference, item.Value, item.TimestampUtc, item.StatusCode); + _last[item.FullReference] = dep; + if (subscribers.Count == 0) continue; // retained for a returning subscriber; nobody to forward to now perSubscriber ??= new Dictionary>(); foreach (var sub in subscribers) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs index 3d2a3058..63ea8736 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs @@ -7,6 +7,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Observability; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry; namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; @@ -76,6 +77,10 @@ public sealed class VirtualTagActor : ReceiveActor private readonly IReadOnlyList _dependencyRefs; private readonly IActorRef? _mux; private readonly ITelemetryLocalHub? _telemetryHub; + // R02:P6 — a script failing on every trigger of a 1 Hz input published one cluster-wide DPS + // ScriptLogEntry per second per tag; 200 such tags shed everyone's entries out of the bounded + // telemetry channels. The per-failure entry is now rate-limited per tag (20/min + one summary). + private readonly ScriptLogRateLimiter _failureLogLimiter; private readonly ILoggingAdapter _log = Context.GetLogger(); // Dependency SNAPSHOTS, not bare values (archreview 02/S14): the status code is what distinguishes a // comms-lost input from a genuine Good null, and it is what the script reads through @@ -104,6 +109,8 @@ public sealed class VirtualTagActor : ReceiveActor /// Optional reference to a dependency multiplexer actor. /// Optional node-local live-telemetry hub each script-log entry is also /// emitted into (Phase 5); null (tests) simply skips the emit. + /// Rate limiter for this tag's per-failure script-log entries; null ⇒ + /// the default 20 per minute + one summary. /// A configured for creating the actor. public static Props Props( string virtualTagId, @@ -113,7 +120,8 @@ public sealed class VirtualTagActor : ReceiveActor Func? publisherFactory = null, IReadOnlyList? dependencyRefs = null, IActorRef? mux = null, - ITelemetryLocalHub? telemetryHub = null) => + ITelemetryLocalHub? telemetryHub = null, + ScriptLogRateLimiter? failureLogLimiter = null) => Akka.Actor.Props.Create(() => new VirtualTagActor( virtualTagId, expression, evaluator ?? NullVirtualTagEvaluator.Instance, @@ -121,7 +129,8 @@ public sealed class VirtualTagActor : ReceiveActor publisherFactory, dependencyRefs ?? Array.Empty(), mux, - telemetryHub)); + telemetryHub, + failureLogLimiter)); /// Initializes a virtual tag actor with the given configuration and dependencies. /// Unique identifier for the virtual tag. @@ -133,6 +142,8 @@ public sealed class VirtualTagActor : ReceiveActor /// Optional reference to a dependency multiplexer actor. /// Optional node-local live-telemetry hub each script-log entry is also /// emitted into (Phase 5); null (tests) simply skips the emit. + /// Rate limiter for this tag's per-failure script-log entries; null ⇒ + /// the default 20 per minute + one summary. public VirtualTagActor( string virtualTagId, string expression, @@ -141,7 +152,8 @@ public sealed class VirtualTagActor : ReceiveActor Func? publisherFactory, IReadOnlyList dependencyRefs, IActorRef? mux, - ITelemetryLocalHub? telemetryHub = null) + ITelemetryLocalHub? telemetryHub = null, + ScriptLogRateLimiter? failureLogLimiter = null) { _virtualTagId = virtualTagId; _scriptId = scriptId; @@ -151,6 +163,7 @@ public sealed class VirtualTagActor : ReceiveActor _dependencyRefs = dependencyRefs; _mux = mux; _telemetryHub = telemetryHub; + _failureLogLimiter = failureLogLimiter ?? new ScriptLogRateLimiter(); Receive(OnDependencyChanged); Receive(OnDependencyBatch); @@ -162,7 +175,9 @@ public sealed class VirtualTagActor : ReceiveActor { if (_mux is not null && _dependencyRefs.Count > 0) { - _mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self)); + // ReplayRetained: a respawned child (plan edit, supervision restart) evaluates from the mux's + // last-known inputs at once instead of waiting for each to change again. + _mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self, ReplayRetained: true)); } } @@ -353,7 +368,18 @@ public sealed class VirtualTagActor : ReceiveActor private void OnEvaluationFailed(string reason, string level, DateTime timestampUtc) { OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair("outcome", "fail")); - PublishLog(level, reason); + + // Rate-limited per tag (R02:P6): the first 20 failures a minute publish as before; the rest are + // counted and surfaced as ONE summary entry when the window rolls. The metric above and the Bad + // publish below are not limited — only the cluster-wide log fan-out is. + var admission = _failureLogLimiter.Admit(_virtualTagId); + if (admission.SuppressedInPreviousWindow > 0) + { + PublishLog("Warning", + $"{admission.SuppressedInPreviousWindow} further evaluation failure(s) were suppressed from the script log " + + $"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); last: {reason}"); + } + if (admission.Admitted) PublishLog(level, reason); if ((_hasPublished && _lastPublishedStatus == OpcUaStatus.BadInternalError) || !AllDependenciesArrived()) { diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs index 0b20f490..b542871f 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs @@ -82,7 +82,9 @@ public sealed class DraftValidatorCalculationTests DriverInstances = [CalcDriver()], Devices = [Device("dev-1", "di-calc", "engine")], Tags = [CalcTagRow("t-1", "dev-1", "a", Cfg("s1"))], - Scripts = [ScriptRow("s1", "return 42.0;")], + // A read, so the change trigger is effective (a constant change-triggered script with no timer + // is now itself a deploy error — CalculationNoEffectiveTrigger). + Scripts = [ScriptRow("s1", "return (double)ctx.GetTag(\"mb/plc/reg\").Value + 42.0;")], }; DraftValidator.Validate(draft).ShouldNotContain(e => e.Code.StartsWith("Calculation")); @@ -132,6 +134,58 @@ public sealed class DraftValidatorCalculationTests msg.ShouldContain("calc/engine/b"); } + // ── effective-trigger gate (R07:U5) ───────────────────────────────────────────────────── + + [Fact] + public void Change_triggered_tag_with_no_literal_read_and_no_timer_is_rejected() + { + var draft = new DraftSnapshot + { + GenerationId = 1, ClusterId = "c", + DriverInstances = [CalcDriver()], + Devices = [Device("dev-1", "di-calc", "engine")], + Tags = [CalcTagRow("t-1", "dev-1", "a", Cfg("s1"))], + Scripts = [ScriptRow("s1", "return ctx.Now.Hour;")], + }; + + DraftValidator.Validate(draft).ShouldContain(e => + e.Code == "CalculationNoEffectiveTrigger" && e.Context == "t-1" && e.Message.Contains("no literal ctx.GetTag")); + } + + [Fact] + public void Tag_with_change_trigger_off_and_no_timer_is_rejected() + { + var draft = new DraftSnapshot + { + GenerationId = 1, ClusterId = "c", + DriverInstances = [CalcDriver()], + Devices = [Device("dev-1", "di-calc", "engine")], + Tags = [CalcTagRow("t-1", "dev-1", "a", "{\"scriptId\":\"s1\",\"changeTriggered\":false}")], + Scripts = [ScriptRow("s1", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")], + }; + + DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationNoEffectiveTrigger" && e.Context == "t-1"); + } + + [Theory] + [InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":true,\"timerIntervalMs\":1000}", "return ctx.Now.Hour;")] // timer rescues a read-less script + [InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":false,\"timerIntervalMs\":500}", "return 1.0;")] // timer-only tag + [InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":true}", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")] // change trigger with a read + [InlineData("{\"scriptId\":\"s1\"}", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")] // changeTriggered defaults to true + public void A_tag_with_an_effective_trigger_is_accepted(string cfg, string source) + { + var draft = new DraftSnapshot + { + GenerationId = 1, ClusterId = "c", + DriverInstances = [CalcDriver()], + Devices = [Device("dev-1", "di-calc", "engine")], + Tags = [CalcTagRow("t-1", "dev-1", "a", cfg)], + Scripts = [ScriptRow("s1", source)], + }; + + DraftValidator.Validate(draft).ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger"); + } + [Fact] public void Cross_driver_ref_is_terminal_not_an_edge() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs new file mode 100644 index 00000000..88015ffc --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs @@ -0,0 +1,166 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Serilog.Events; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests; + +/// +/// Review 07 Calculation remediation: a compile failure is negative-cached by source hash and its +/// Warning/script-log fires on the Good→Bad transition only (P4); inputs are recorded before +/// InitializeAsync so a Restart computes from current inputs (H6); array results dedup +/// structurally (H8); ShutdownAsync drops the compiled scripts (L2). +/// +public sealed class CalculationDriverRemediationTests +{ + private const uint StatusGood = 0u; + private const uint StatusBadInternalError = 0x80020000u; + + private static RawTagEntry CalcTag(string rawPath, string source, bool changeTriggered = true, int? timerMs = null) + { + var timer = timerMs is { } ms ? $",\"timerIntervalMs\":{ms}" : ""; + var json = $"{{\"scriptId\":\"s-{rawPath}\",\"changeTriggered\":{(changeTriggered ? "true" : "false")}{timer}," + + $"\"scriptSource\":{System.Text.Json.JsonSerializer.Serialize(source)}}}"; + return new RawTagEntry(rawPath, json, WriteIdempotent: false); + } + + private static (CalculationDriver Driver, ConcurrentQueue Published, List Logs, CapturingLogger HostLog) + Build(params RawTagEntry[] tags) + { + var logs = new List(); + var sink = new ScriptLogTopicSink(new CapturingPublisher(logs), LogEventLevel.Information); + var scriptRoot = new ScriptRootLogger(new LoggerConfiguration().WriteTo.Sink(sink).CreateLogger()); + var hostLog = new CapturingLogger(); + var driver = new CalculationDriver(new CalculationDriverOptions { RawTags = tags }, "calc-drv", scriptRoot, hostLog); + var published = new ConcurrentQueue(); + driver.OnDataChange += (_, e) => published.Enqueue(e); + return (driver, published, logs, hostLog); + } + + private static async Task Settle(CalculationDriver driver) + { + var deadline = DateTime.UtcNow.AddSeconds(30); + while (!driver.IsEvaluationIdle && DateTime.UtcNow < deadline) await Task.Delay(10); + driver.IsEvaluationIdle.ShouldBeTrue("the calc driver's evaluation pump should have drained"); + } + + [Fact] + public async Task A_compile_failure_is_remembered_and_logged_once_across_repeated_triggers() + { + var (driver, published, logs, hostLog) = Build( + CalcTag("calc/e/broken", "var x = (double)ctx.GetTag(\"src/a\").Value; this is not valid C#;")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + for (var i = 0; i < 5; i++) + { + driver.OnDependencyValue("src/a", i, StatusGood, DateTime.UtcNow); + await Settle(driver); + } + + published.Count.ShouldBe(1, "Bad is published once per transition"); + published.TryDequeue(out var bad).ShouldBeTrue(); + bad!.Snapshot.StatusCode.ShouldBe(StatusBadInternalError); + logs.Count.ShouldBe(1, "the script-log entry fires on the Good→Bad transition only"); + hostLog.Entries.Count(e => e.Level == LogLevel.Warning && e.Message.Contains("compile failed")).ShouldBe(1, + "Roslyn is consulted once per source; later triggers hit the negative cache"); + driver.CompileFailureCountForTest.ShouldBe(1); + } + + [Fact] + public async Task A_changed_failure_reason_is_logged_even_while_already_Bad() + { + // Throws with a different message depending on the input, so the failure REASON changes while the + // published status stays Bad — the operator must see the new reason. + var (driver, published, logs, _) = Build( + CalcTag("calc/e/thrower", "throw new System.InvalidOperationException(\"boom \" + (int)ctx.GetTag(\"src/a\").Value);")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); + await Settle(driver); + driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); // same reason → no new log + await Settle(driver); + driver.OnDependencyValue("src/a", 2, StatusGood, DateTime.UtcNow); // new reason → logged + await Settle(driver); + + published.Count.ShouldBe(1); + logs.Count.ShouldBe(2); + logs[0].Message.ShouldContain("boom 1"); + logs[1].Message.ShouldContain("boom 2"); + } + + [Fact] + public async Task Inputs_delivered_before_InitializeAsync_are_kept_so_the_first_trigger_computes() + { + // H6: a Restart is a new driver whose adapter registers interest at construction; the mux's + // retained-value replay therefore lands BEFORE InitializeAsync. It used to be dropped. + var (driver, published, _, _) = Build( + CalcTag("calc/e/sum", "return (double)ctx.GetTag(\"src/a\").Value + (double)ctx.GetTag(\"src/b\").Value;")); + + driver.OnDependencyValue("src/a", 10.0, StatusGood, DateTime.UtcNow); // replayed, pre-init + driver.OnDependencyValue("src/b", 20.0, StatusGood, DateTime.UtcNow); + published.ShouldBeEmpty("evaluation is gated on running"); + + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + driver.OnDependencyValue("src/a", 12.0, StatusGood, DateTime.UtcNow); // only ONE input moves after init + await Settle(driver); + + published.Count.ShouldBe(1, "src/b's pre-init value satisfied the change-gate"); + published.TryDequeue(out var e).ShouldBeTrue(); + e!.Snapshot.Value.ShouldBe(32.0); + } + + [Fact] + public async Task An_equal_array_result_is_deduped_structurally() + { + var (driver, published, _, _) = Build( + CalcTag("calc/e/arr", "var v = (double)ctx.GetTag(\"src/a\").Value; return new double[] { v, v * 2 };")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); + await Settle(driver); + driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); // same array contents + await Settle(driver); + published.Count.ShouldBe(1, "a structurally equal array is not a change"); + + driver.OnDependencyValue("src/a", 2.0, StatusGood, DateTime.UtcNow); + await Settle(driver); + published.Count.ShouldBe(2, "a different array still publishes"); + published.Last().Snapshot.Value.ShouldBe(new double[] { 2.0, 4.0 }); + } + + [Fact] + public async Task ShutdownAsync_drops_the_compiled_scripts() + { + var (driver, published, _, _) = Build(CalcTag("calc/e/x", "return (double)ctx.GetTag(\"src/a\").Value * 2.0;")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); + await Settle(driver); + published.Count.ShouldBe(1); + driver.CompiledScriptCountForTest.ShouldBe(1); + + await driver.ShutdownAsync(TestContext.Current.CancellationToken); + + driver.CompiledScriptCountForTest.ShouldBe(0, "the collectible ALCs must not survive the generation"); + } + + private sealed class CapturingPublisher(List entries) : IScriptLogPublisher + { + public void Publish(ScriptLogEntry entry) { lock (entries) entries.Add(entry); } + } + + internal sealed class CapturingLogger : ILogger + { + public ConcurrentBag<(LogLevel Level, string Message)> Entries { get; } = new(); + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, + Func formatter) + => Entries.Add((logLevel, formatter(state, exception))); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs index 5eed4c6b..cf22038b 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs @@ -229,11 +229,13 @@ public sealed class CalculationDriverTests logs[0].ScriptId.ShouldBe("s-calc/e/div"); logs[0].VirtualTagId.ShouldBe("calc/e/div"); - // a = 0 again → still failing, but NOT a fresh transition → no second Bad publish (script-log still emits). + // a = 0 again → still failing for the SAME reason, NOT a fresh transition → no second Bad publish, + // and (R07:P4) no second script-log entry either: the log fires on the transition or on a change of + // reason, not on every trigger of a tag that is already Bad. driver.OnDependencyValue("src/a", 0, StatusGood, DateTime.UtcNow); await Settle(driver); published.ShouldBeEmpty(); - logs.Count.ShouldBe(2); + logs.Count.ShouldBe(1); // a = 5 → 20 → Good recovery is force-published. driver.OnDependencyValue("src/a", 5, StatusGood, DateTime.UtcNow); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs index d020d118..a3861648 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs @@ -15,6 +15,28 @@ public sealed class DependencyMuxTagUpstreamSourceTests private static DataValueSnapshot Good(object? value, DateTime ts) => new(value, 0u, ts, ts); + /// R02:L3 — the host prunes the retained snapshots to the loaded dependency-ref union after each + /// (re)load, so a renamed/removed raw path stops being retained; observers are untouched. + [Fact] + public void RetainOnly_drops_snapshots_outside_the_set_and_keeps_observers() + { + var src = new DependencyMuxTagUpstreamSource(); + var ts = DateTime.UtcNow; + src.Push("keep", Good(1, ts)); + src.Push("old/renamed", Good(2, ts)); + var seen = new List(); + using var sub = src.SubscribeTag("old/renamed", (p, _) => seen.Add(p)); + + src.RetainOnly(new HashSet(StringComparer.Ordinal) { "keep" }).ShouldBe(1); + + src.RetainedPathCount.ShouldBe(1); + src.ReadTag("keep").Value.ShouldBe(1); + src.ReadTag("old/renamed").StatusCode.ShouldBe(OpcUaStatus.BadWaitingForInitialData); + src.Push("old/renamed", Good(3, ts)); + // Pruning retained values does not unsubscribe anyone. + seen.ShouldBe(new[] { "old/renamed" }); + } + [Fact] public void Push_then_ReadTag_returns_the_pushed_snapshot() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs new file mode 100644 index 00000000..c4063d57 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs @@ -0,0 +1,145 @@ +using Akka.Actor; +using Akka.TestKit; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; +using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.ScriptedAlarms; + +/// +/// R02:H1 — a supervision restart of used to dispose the +/// Props-captured engine and re-create the actor around the disposed instance, so every later +/// ApplyScriptedAlarms failed and the node's scripted alarms silently died. With the factory +/// Props each incarnation builds its own engine; with the legacy Props a restart only detaches. Both +/// shapes must load cleanly after a throw inside a +/// handler. Also R02:H3 — an inbound alarm command's engine op is bounded. +/// +public sealed class ScriptedAlarmHostActorRestartTests : RuntimeActorTestBase +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); + + private static EquipmentScriptedAlarmPlan Plan(string id = "alm-1", string depRef = "M.T") => + new(ScriptedAlarmId: id, EquipmentId: "Plant/Line1/M", Name: "HighTemp", AlarmType: "AlarmCondition", + Severity: 800, MessageTemplate: "condition", PredicateScriptId: $"{id}-script", + PredicateSource: $"return (int)ctx.GetTag(\"{depRef}\").Value > 90;", + DependencyRefs: new[] { depRef }, HistorizeToAveva: true, Retain: true, Enabled: true); + + private static ScriptedAlarmEngine BuildEngine(DependencyMuxTagUpstreamSource upstream, IAlarmStateStore? store = null) + { + var logger = new LoggerConfiguration().CreateLogger(); + return new ScriptedAlarmEngine(upstream, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger); + } + + private static (DependencyMuxTagUpstreamSource, ScriptedAlarmEngine) Pair() + { + var up = new DependencyMuxTagUpstreamSource(); + return (up, BuildEngine(up)); + } + + /// Apply, crash the handler (a null TagId makes Push throw → restart), apply again + /// and prove the second load registers interest and the alarm still activates. + private void RestartThenReload(IActorRef host, TestProbe publish, TestProbe mux) + { + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() })); + mux.ExpectMsg(Timeout); + + // The crash: a batch whose item has a null TagId makes the upstream Push throw out of the handler + // (the documented, deliberately unguarded path) → supervision restarts the actor. + host.Tell(new VirtualTagActor.DependencyValueBatch(new[] + { + new VirtualTagActor.DependencyValueChanged(null!, 1, DateTime.UtcNow), + })); + mux.ExpectMsg(Timeout); // PostStop of the crashed incarnation + + // A later apply must load cleanly on the restarted incarnation… + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() })); + var reg = mux.ExpectMsg(Timeout); + reg.TagRefs.ShouldContain("M.T"); + + // …and the alarm must still work end to end. + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow)); + var state = publish.ExpectMsg(Timeout); + state.AlarmNodeId.ShouldBe("alm-1"); + state.State.Active.ShouldBeTrue(); + } + + [Fact] + public void Factory_Props_survives_a_handler_throw_and_loads_cleanly_afterwards() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var built = 0; + var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, () => { built++; return Pair(); })); + + RestartThenReload(host, publish, mux); + + built.ShouldBe(2, "the restarted incarnation built its own engine"); + } + + [Fact] + public void Legacy_instance_Props_survives_a_handler_throw_because_a_restart_only_detaches() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var (upstream, engine) = Pair(); + var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, upstream, engine)); + + RestartThenReload(host, publish, mux); + } + + [Fact] + public void An_alarm_command_whose_store_hangs_is_bounded_and_the_mailbox_moves_on() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var store = new HangingSaveStore(); + var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, + () => { var up = new DependencyMuxTagUpstreamSource(); return (up, BuildEngine(up, store)); }, + commandTimeout: TimeSpan.FromMilliseconds(300))); + + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() })); + mux.ExpectMsg(Timeout); + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow)); + publish.ExpectMsg(Timeout).State.Active.ShouldBeTrue(); + + store.HangNext = true; + host.Tell(new AlarmCommand("alm-1", "Acknowledge", "alice", "ack", null)); + + // The hung save would have suspended the mailbox indefinitely; the bound releases it, so the + // next dependency value is processed and the alarm clears. + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 10, DateTime.UtcNow)); + publish.ExpectMsg(Timeout).State.Active.ShouldBeFalse(); + store.ObservedCancellation.ShouldBeTrue(); + } + + /// Store whose next SaveAsync blocks until its token is cancelled. + private sealed class HangingSaveStore : IAlarmStateStore + { + private readonly InMemoryAlarmStateStore _inner = new(); + public volatile bool HangNext; + public volatile bool ObservedCancellation; + + public Task LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct); + public Task> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct); + public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct); + + public async Task SaveAsync(AlarmConditionState state, CancellationToken ct) + { + if (HangNext) + { + HangNext = false; + try { await Task.Delay(System.Threading.Timeout.InfiniteTimeSpan, ct); } + catch (OperationCanceledException) { ObservedCancellation = true; throw; } + } + await _inner.SaveAsync(state, ct); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs new file mode 100644 index 00000000..ae42edb9 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs @@ -0,0 +1,101 @@ +using System.Collections.Concurrent; +using Akka.Actor; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Calculation; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags; + +/// +/// R07:H6 — the mux retains the last value per ever-registered ref and replays it to a registration +/// that asks (). A Calculation driver +/// Restart (new driver + new adapter) therefore republishes from current inputs instead of sitting +/// BadWaitingForInitialData until every input happens to change. +/// +public sealed class DependencyMuxReplayTests : RuntimeActorTestBase +{ + private static DriverInstanceActor.AttributeValueBatchPublished Burst(params (string Ref, object? Value)[] items) => + new("driver-1", items.Select(i => new DriverInstanceActor.AttributeValueItem(i.Ref, i.Value, OpcUaStatus.Good, DateTime.UtcNow)).ToArray()); + + [Fact] + public void RegisterInterest_with_replay_receives_the_retained_values_for_its_refs_only() + { + var mux = Sys.ActorOf(DependencyMuxActor.Props()); + var first = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, first.Ref)); + mux.Tell(Burst(("a", 1), ("b", 2), ("c", 3))); + first.ExpectMsg(PresenceBudget).Items.Count.ShouldBe(2); + + // The first subscriber goes away; retention must survive the unregister (that is the Restart shape). + mux.Tell(new DependencyMuxActor.UnregisterInterest(first.Ref)); + + var second = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "c" }, second.Ref, ReplayRetained: true)); + + var replay = second.ExpectMsg(PresenceBudget); + replay.Items.Select(i => i.TagId).ShouldBe(new[] { "a" }, "'c' was never registered by anyone, so it is not retained; 'b' is not in this subscriber's refs"); + replay.Items[0].Value.ShouldBe(1); + } + + [Fact] + public void RegisterInterest_without_replay_receives_nothing_and_a_fresh_ref_replays_nothing() + { + var mux = Sys.ActorOf(DependencyMuxActor.Props()); + var seed = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, seed.Ref)); + mux.Tell(Burst(("a", 1))); + seed.ExpectMsg(PresenceBudget); + + var opted = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, opted.Ref)); // default: no replay + opted.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + + var fresh = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "never-published" }, fresh.Ref, ReplayRetained: true)); + fresh.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + } + + /// The user-facing scenario: a calc tag computes, the driver is restarted (new instance + new + /// adapter) with NO input changing afterwards, and the new instance republishes the current value. + [Fact] + public async Task A_Calculation_driver_restart_republishes_from_the_current_inputs() + { + var mux = Sys.ActorOf(DependencyMuxActor.Props()); + var tag = new RawTagEntry("calc/e/sum", + "{\"scriptId\":\"s\",\"changeTriggered\":true,\"scriptSource\":\"return (double)ctx.GetTag(\\\"src/a\\\").Value + (double)ctx.GetTag(\\\"src/b\\\").Value;\"}", + WriteIdempotent: false); + + // Generation 1: wire the driver through the real adapter and feed a burst. + var gen1 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv"); + var published1 = new ConcurrentQueue(); + gen1.OnDataChange += (_, e) => published1.Enqueue(e); + await gen1.InitializeAsync("{}", CancellationToken.None); + var adapter1 = Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen1, mux)); + AwaitAssert(() => + { + mux.Tell(Burst(("src/a", 10.0), ("src/b", 20.0))); + published1.Count.ShouldBeGreaterThan(0); + }, PresenceBudget); + published1.Last().Snapshot.Value.ShouldBe(30.0); + + // Restart: the old instance goes away; a NEW driver + adapter come up. Nothing changes upstream. + Sys.Stop(adapter1); + await gen1.ShutdownAsync(CancellationToken.None); + var gen2 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv"); + var published2 = new ConcurrentQueue(); + gen2.OnDataChange += (_, e) => published2.Enqueue(e); + Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen2, mux)); // registers with ReplayRetained + await gen2.InitializeAsync("{}", CancellationToken.None); + + // The replayed inputs satisfied the change-gate; the first trigger after init is the replay itself + // (if it lands post-init) or the next change (here: none) — so publish happens from the replay. + AwaitAssert(() => published2.Count.ShouldBeGreaterThan(0), PresenceBudget); + published2.Last().Snapshot.Value.ShouldBe(30.0); + published2.Last().Snapshot.StatusCode.ShouldBe(OpcUaStatus.Good); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs new file mode 100644 index 00000000..74ee2e32 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs @@ -0,0 +1,55 @@ +using System.Collections.Concurrent; +using Akka.Actor; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Engines; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags; + +/// R02:P6 — the per-failure cluster-wide is rate-limited per tag +/// (first N per window, then one summary); the Bad publish and the metric are unaffected. +public sealed class VirtualTagActorFailureLogRateLimitTests : RuntimeActorTestBase +{ + [Fact] + public void Repeated_failures_publish_at_most_the_limit_then_one_summary_when_the_window_rolls() + { + var now = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc); + var limiter = new ScriptLogRateLimiter(limitPerWindow: 3, window: TimeSpan.FromMinutes(1), clock: () => now); + var capture = new ConcurrentQueue(); + var parent = CreateTestProbe(); + var actor = parent.ChildActorOf(VirtualTagActor.Props( + "vt-chatty", "broken", + evaluator: new FailingEvaluator("boom"), + publisherFactory: () => new DPSPublisher((_, p) => capture.Enqueue((ScriptLogEntry)p)), + failureLogLimiter: limiter)); + + for (var i = 0; i < 10; i++) + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", i, DateTime.UtcNow)); + + // Positive control: the FIRST failure still degrades the node (once per transition, as before). + parent.ExpectMsg(PresenceBudget).StatusCode.ShouldBe(Commons.OpcUa.OpcUaStatus.BadInternalError); + AwaitAssert(() => capture.Count.ShouldBe(3), PresenceBudget); + parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + capture.Count.ShouldBe(3, "7 of 10 failures were suppressed from the cluster-wide log"); + + now = now.AddMinutes(1); + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 99, DateTime.UtcNow)); + + AwaitAssert(() => capture.Count.ShouldBe(5), PresenceBudget); + var entries = capture.ToArray(); + entries[3].Message.ShouldContain("7"); + entries[3].Message.ShouldContain("suppressed"); + entries[3].VirtualTagId.ShouldBe("vt-chatty"); + entries[4].Message.ShouldBe("boom"); + } + + private sealed class FailingEvaluator(string reason) : IVirtualTagEvaluator + { + public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary deps) + => VirtualTagEvalResult.Failure(reason); + } +} From 1bf5ef916cedd6ab532edca25e6492320df048e3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:27:59 -0400 Subject: [PATCH 10/27] feat(controlplane): deployment artifact retention on the admin singleton MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every deploy stored the whole gzip'd fleet snapshot in Deployment.ArtifactBlob forever (R03:1.2). DeploymentRetentionActor (admin singleton, proxy-less) sweeps every Deployments:Retention: IntervalMinutes (60): terminal rows (Sealed / PartiallyFailed / TimedOut) older than the newest KeepSealed (50) and not referenced by a NodeDeploymentState started/applied within ReferencedDays (30) get their blob EMPTIED — never deleted, never NULLed: the column stays NOT NULL and every reader already treats zero bytes as "no artifact" (#485), so no schema migration and no reader change. In-flight rows are never touched; MaxRowsPerSweep (200) bounds one sweep. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Entities/Deployment.cs | 6 +- .../Retention/DeploymentRetentionActor.cs | 177 ++++++++++++++++++ .../Retention/DeploymentRetentionOptions.cs | 54 ++++++ .../ServiceCollectionExtensions.cs | 18 ++ .../DeploymentRetentionTests.cs | 174 +++++++++++++++++ 5 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs index 19d83728..d3044244 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs @@ -26,7 +26,11 @@ public sealed class Deployment public DateTime CreatedAtUtc { get; init; } = DateTime.UtcNow; /// Gets or sets the serialized artifact blob containing the configuration. - public byte[] ArtifactBlob { get; init; } = Array.Empty(); + /// + /// Settable (not init-only) so the retention sweep can EMPTY it on an old terminal row. An empty blob + /// is "no artifact" to every reader (#485), so the audit row survives with its bytes retired. + /// + public byte[] ArtifactBlob { get; set; } = Array.Empty(); /// Gets or sets the row version for optimistic concurrency control. public byte[] RowVersion { get; set; } = Array.Empty(); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs new file mode 100644 index 00000000..55ffd7dc --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs @@ -0,0 +1,177 @@ +using Akka.Actor; +using Akka.Event; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; + +/// +/// Admin-role cluster singleton that periodically empties the ArtifactBlob of old, terminal +/// deployments (R03:1.2). See for the policy and +/// for the rule, which is pure over a context so it is unit-testable. +/// +/// +/// +/// Empties, never deletes, and never nulls. The column is NOT NULL varbinary(max), and +/// every reader (DriverHostActor.LoadArtifactBlob, OpcUaPublishActor, +/// DeploymentArtifactService) already treats a zero-length blob as "no artifact" (#485: empty +/// bytes are not an empty configuration — the apply fails and the node keeps last-known-good). An +/// empty blob therefore needs no schema change and no reader change, where a NULL would need both. +/// +/// +/// Never touches an in-flight row (Dispatching / AwaitingApplyAcks): a node may still be +/// fetching it. The newest terminal rows keep +/// their blob whatever their age, so the "latest sealed" every boot path reads is always intact. +/// +/// +public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers +{ + private readonly IDbContextFactory _dbFactory; + private readonly DeploymentRetentionOptions _options; + private readonly ILoggingAdapter _log = Context.GetLogger(); + private const string SweepTimerKey = "retention-sweep"; + + /// Gets or sets the timer scheduler. + public ITimerScheduler Timers { get; set; } = null!; + + /// Creates the actor props. + /// The config DB factory. + /// The retention policy. + /// The props. + public static Props Props(IDbContextFactory dbFactory, DeploymentRetentionOptions options) => + Akka.Actor.Props.Create(() => new DeploymentRetentionActor(dbFactory, options)); + + /// Initializes a new instance of the class. + /// The config DB factory. + /// The retention policy. + public DeploymentRetentionActor(IDbContextFactory dbFactory, DeploymentRetentionOptions options) + { + ArgumentNullException.ThrowIfNull(dbFactory); + ArgumentNullException.ThrowIfNull(options); + _dbFactory = dbFactory; + _options = options; + + Receive(_ => RunSweep()); + Receive(msg => + { + if (msg.Pruned > 0) + _log.Info("Deployment retention emptied the artifact blob on {Count} old deployment(s)", msg.Pruned); + else + _log.Debug("Deployment retention sweep found nothing to prune"); + if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg); + }); + Receive(msg => + { + _log.Warning(msg.Error, "Deployment retention sweep failed; the next sweep retries"); + if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg); + }); + } + + /// + protected override void PreStart() + { + if (!_options.Enabled) + { + _log.Info("Deployment retention is disabled (Deployments:Retention:Enabled=false); artifact blobs are kept forever"); + return; + } + var interval = TimeSpan.FromMinutes(Math.Max(1, _options.IntervalMinutes)); + var initial = TimeSpan.FromSeconds(Math.Max(0, _options.InitialDelaySeconds)); + Timers.StartPeriodicTimer(SweepTimerKey, Sweep.Instance, initial, interval); + } + + private void RunSweep() + { + var replyTo = Sender.IsNobody() ? null : Sender; + var self = Self; + var now = DateTime.UtcNow; + Task.Run(async () => + { + await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false); + return await SweepAsync(db, _options, now).ConfigureAwait(false); + }) + .PipeTo(self, + success: pruned => new SweepCompleted(pruned, replyTo), + failure: ex => new SweepFailed(ex, replyTo)); + } + + /// + /// The retention rule. Empties the blob of every terminal deployment that is (a) older than the + /// newest terminal rows, (b) not referenced by a + /// NodeDeploymentState row started or applied within + /// , and (c) still carrying bytes — at most + /// per call. + /// + /// The config DB. + /// The policy. + /// The sweep's clock. + /// Cancellation. + /// How many rows were emptied. + public static async Task SweepAsync( + OtOpcUaConfigDbContext db, DeploymentRetentionOptions options, DateTime utcNow, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(options); + + var keep = Math.Max(1, options.KeepSealed); + var referenceFloor = utcNow.AddDays(-Math.Max(0, options.ReferencedDays)); + var batch = Math.Max(1, options.MaxRowsPerSweep); + + // Projected: ids only. The candidate scan must not materialise blobs for rows it will keep. + var terminal = await db.Deployments + .AsNoTracking() + .Where(d => d.Status == DeploymentStatus.Sealed + || d.Status == DeploymentStatus.PartiallyFailed + || d.Status == DeploymentStatus.TimedOut) + .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, HasBlob = d.ArtifactBlob.Length > 0 }) + .ToListAsync(ct) + .ConfigureAwait(false); + + var aged = terminal.Skip(keep).Where(d => d.HasBlob).Select(d => d.DeploymentId).ToList(); + if (aged.Count == 0) return 0; + + var referenced = (await db.NodeDeploymentStates + .AsNoTracking() + .Where(s => aged.Contains(s.DeploymentId) + && (s.StartedAtUtc >= referenceFloor + || (s.AppliedAtUtc != null && s.AppliedAtUtc >= referenceFloor))) + .Select(s => s.DeploymentId) + .Distinct() + .ToListAsync(ct) + .ConfigureAwait(false)) + .ToHashSet(); + + var targets = aged.Where(id => !referenced.Contains(id)).Take(batch).ToList(); + if (targets.Count == 0) return 0; + + // Tracked load of ONLY the rows being emptied: the update rides the row's real RowVersion on + // both providers, and the blob read is the one-time cost of retiring that row. + var rows = await db.Deployments + .Where(d => targets.Contains(d.DeploymentId)) + .ToListAsync(ct) + .ConfigureAwait(false); + foreach (var row in rows) row.ArtifactBlob = []; + await db.SaveChangesAsync(ct).ConfigureAwait(false); + return rows.Count; + } + + /// Asks for a sweep now; the sender (if any) receives the / . + public sealed class Sweep + { + /// The singleton instance. + public static readonly Sweep Instance = new(); + private Sweep() { } + } + + /// A sweep finished. + /// How many rows were emptied. + /// Who asked, when the sweep was requested rather than timer-driven. + public sealed record SweepCompleted(int Pruned, IActorRef? ReplyTo); + + /// A sweep threw. + /// The failure. + /// Who asked, when the sweep was requested rather than timer-driven. + public sealed record SweepFailed(Exception Error, IActorRef? ReplyTo); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs new file mode 100644 index 00000000..6654ec1e --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; + +/// +/// Deployments:Retention — how long a sealed deployment keeps its ArtifactBlob (R03:1.2). +/// Every deploy stored the whole gzip'd fleet snapshot forever; nothing pruned it, so a year of routine +/// edits was gigabytes of blobs. The retention sweep EMPTIES the blob on old rows and never deletes a +/// row: the audit record (who deployed what revision, when, and how every node acked) survives. +/// +public sealed class DeploymentRetentionOptions +{ + /// The configuration section name. + public const string SectionName = "Deployments:Retention"; + + /// Master switch; keeps every blob forever (the previous behaviour). + public bool Enabled { get; set; } = true; + + /// + /// How many of the most recent terminal (Sealed / PartiallyFailed / TimedOut) deployments keep their + /// blob regardless of age. A node booting from central SQL reads the LATEST sealed row's blob, so this + /// must stay well above 1; 50 also keeps a comfortable re-deploy history. + /// + public int KeepSealed { get; set; } = 50; + + /// + /// A deployment any NodeDeploymentState row started or applied within this many days keeps its + /// blob even when older than the window — a node that just applied it may + /// still replay it on restart. + /// + public int ReferencedDays { get; set; } = 30; + + /// How often the sweep runs. + public int IntervalMinutes { get; set; } = 60; + + /// Delay before the first sweep after the singleton starts, so a boot is not a blob scan. + public int InitialDelaySeconds { get; set; } = 60; + + /// + /// Rows emptied per sweep. The sweep loads each candidate row (blob included) to update it under its + /// RowVersion, so this bounds one sweep's memory and transaction size; a backlog drains over sweeps. + /// + public int MaxRowsPerSweep { get; set; } = 200; + + /// Binds the section from configuration, with defaults for anything absent. + /// The app configuration; null yields the defaults. + /// The bound options. + public static DeploymentRetentionOptions Bind(IConfiguration? configuration) + { + var options = new DeploymentRetentionOptions(); + configuration?.GetSection(SectionName).Bind(options); + return options; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index cb8d605a..f838a004 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -15,6 +15,7 @@ using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.ControlPlane; @@ -30,6 +31,7 @@ public static class ServiceCollectionExtensions public const string AuditWriterSingletonName = "audit-writer"; public const string RedundancyStateSingletonName = "redundancy-state"; public const string ClusterNodeAddressReconcilerSingletonName = "cluster-node-address-reconciler"; + public const string DeploymentRetentionSingletonName = "deployment-retention"; /// /// Registers the admin-role cluster singletons on the AkkaConfigurationBuilder — three with @@ -181,6 +183,21 @@ public static class ServiceCollectionExtensions singletonOptions, createProxyToo: false); + // Deployment artifact retention (R03:1.2): empties ArtifactBlob on old terminal deployments so the + // Deployment table stops growing by one fleet snapshot per edit. Admin singleton for the same + // reason as the reconciler (one sweeper per fleet, and only central holds the ConfigDb); + // proxy-less because nothing addresses it in production — a test sends Sweep directly. + builder.WithSingleton( + DeploymentRetentionSingletonName, + (system, registry, resolver) => + { + var dbFactory = resolver.GetService>(); + var options = DeploymentRetentionOptions.Bind(resolver.GetService()); + return DeploymentRetentionActor.Props(dbFactory, options); + }, + singletonOptions, + createProxyToo: false); + return builder; } @@ -256,4 +273,5 @@ public sealed class AdminOperationsActorKey { } public sealed class AuditWriterActorKey { } public sealed class RedundancyStateActorKey { } public sealed class ClusterNodeAddressReconcilerKey { } +public sealed class DeploymentRetentionKey { } public sealed class CentralCommunicationActorKey { } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs new file mode 100644 index 00000000..ecd397c4 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs @@ -0,0 +1,174 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:1.2 — every deploy stored the whole fleet snapshot forever. The retention sweep empties the +/// blob on old terminal rows (never deletes a row, never touches an in-flight one, never the newest N, +/// never one a node applied recently). +/// +public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase +{ + private static readonly DateTime SweepNow = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc); + + [Fact] + public async Task Sweep_empties_blobs_beyond_the_keep_window_and_leaves_the_newest_intact() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 12); // ids[0] newest … ids[11] oldest + var options = new DeploymentRetentionOptions { KeepSealed = 5, ReferencedDays = 30, MaxRowsPerSweep = 200 }; + + int pruned; + await using (var db = f.CreateDbContext()) + pruned = await DeploymentRetentionActor.SweepAsync(db, options, SweepNow); + + pruned.ShouldBe(7); + await using var verify = f.CreateDbContext(); + foreach (var id in ids.Take(5)) + (await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.Length.ShouldBeGreaterThan(0, "newest 5 keep their blob"); + foreach (var id in ids.Skip(5)) + (await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.ShouldBeEmpty("older rows are emptied"); + // Rows survive: the audit record is intact. + (await verify.Deployments.CountAsync()).ShouldBe(12); + + // Idempotent: a second sweep finds nothing. + await using var again = f.CreateDbContext(); + (await DeploymentRetentionActor.SweepAsync(again, options, SweepNow)).ShouldBe(0); + } + + [Fact] + public async Task Sweep_keeps_a_row_a_node_applied_recently_and_never_touches_an_in_flight_row() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 4); + var inFlight = Guid.NewGuid(); + await using (var db = f.CreateDbContext()) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1", + ApplicationUri = "urn:x", Enabled = true, CreatedBy = "test", + }); + // ids[3] is the oldest; a node applied it yesterday, so it stays. + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = "site-a-1:4053", DeploymentId = ids[3], Status = NodeDeploymentStatus.Applied, + StartedAtUtc = SweepNow.AddDays(-1), AppliedAtUtc = SweepNow.AddDays(-1), + }); + // ids[2]: applied long ago — not protected. + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = "site-a-1:4053", DeploymentId = ids[2], Status = NodeDeploymentStatus.Applied, + StartedAtUtc = SweepNow.AddDays(-90), AppliedAtUtc = SweepNow.AddDays(-90), + }); + // An ancient in-flight row with a blob — never touched. + db.Deployments.Add(new Deployment + { + DeploymentId = inFlight, RevisionHash = new string('f', 64), Status = DeploymentStatus.AwaitingApplyAcks, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-400), ArtifactBlob = new byte[] { 1, 2, 3 }, + }); + await db.SaveChangesAsync(); + } + var options = new DeploymentRetentionOptions { KeepSealed = 1, ReferencedDays = 30 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2); // ids[1], ids[2] + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty("newest"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[1])).ArtifactBlob.ShouldBeEmpty(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[2])).ArtifactBlob.ShouldBeEmpty("applied 90 days ago"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[3])).ArtifactBlob.ShouldNotBeEmpty("applied yesterday"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == inFlight)).ArtifactBlob.ShouldNotBeEmpty("in flight"); + } + + [Fact] + public async Task Sweep_is_bounded_per_call_and_drains_a_backlog_over_calls() + { + var f = NewInMemoryDbFactory(); + Seed(f, sealedCount: 10); + var options = new DeploymentRetentionOptions { KeepSealed = 2, MaxRowsPerSweep = 3 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(0); + } + + /// The actor runs the same rule off its thread and reports back. + [Fact] + public void Actor_sweeps_on_request_and_reports_the_count() + { + var f = NewInMemoryDbFactory(); + Seed(f, sealedCount: 6); + var actor = Sys.ActorOf(DeploymentRetentionActor.Props(f, new DeploymentRetentionOptions + { + KeepSealed = 4, InitialDelaySeconds = 3600, IntervalMinutes = 60, + })); + + actor.Tell(DeploymentRetentionActor.Sweep.Instance); + + ExpectMsg(PresenceBudget).Pruned.ShouldBe(2); + using var verify = f.CreateDbContext(); + verify.Deployments.Count(d => d.ArtifactBlob.Length == 0).ShouldBe(2); + } + + [Fact] + public void Options_bind_from_configuration_with_defaults() + { + var defaults = DeploymentRetentionOptions.Bind(null); + defaults.Enabled.ShouldBeTrue(); + defaults.KeepSealed.ShouldBe(50); + defaults.ReferencedDays.ShouldBe(30); + defaults.IntervalMinutes.ShouldBe(60); + + var config = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Deployments:Retention:KeepSealed"] = "7", + ["Deployments:Retention:Enabled"] = "false", + }) + .Build(); + var bound = DeploymentRetentionOptions.Bind(config); + bound.KeepSealed.ShouldBe(7); + bound.Enabled.ShouldBeFalse(); + bound.ReferencedDays.ShouldBe(30); + } + + /// Seeds sealed deployments, newest first; each one hour older than the previous. + private static List Seed(IDbContextFactory f, int sealedCount) + { + var ids = new List(); + using var db = f.CreateDbContext(); + for (var i = 0; i < sealedCount; i++) + { + var id = Guid.NewGuid(); + ids.Add(id); + db.Deployments.Add(new Deployment + { + DeploymentId = id, + RevisionHash = new string('a', 64), + Status = i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed, + CreatedBy = "test", + CreatedAtUtc = SweepNow.AddHours(-i), + SealedAtUtc = SweepNow.AddHours(-i), + ArtifactBlob = new byte[] { 0x1f, 0x8b, (byte)i }, + }); + } + db.SaveChanges(); + return ids; + } +} From 18352442ab93f207acba2986a0c310968490578c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:30:03 -0400 Subject: [PATCH 11/27] fix(controlplane): audit writer drops only the rejected row, not the whole batch AuditWriterActor documented per-row dedup on the UX_ConfigAuditLog_EventId index but flushed one statement batch: a single duplicate EventId (any at-least-once producer across a singleton hand-over) threw for all 500 rows (R03:2.6). SaveChanges now detaches the entries a DbUpdateException names (2601/2627 logged at Debug as the expected duplicate, anything else at Warning) and re-saves, bounded by the batch size; a failure naming no entry stays a whole-batch failure. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Audit/AuditWriterActor.cs | 57 ++++++++- .../AuditWriterActorRejectedRowTests.cs | 110 ++++++++++++++++++ 2 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs index 2ca3026c..0e20da2f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs @@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; /// Dedup is two-layer: in-buffer (the below collapses /// duplicate EventIds before flush) and at the database via the filtered unique index /// UX_ConfigAuditLog_EventId (cross-restart safety — a retry of an already-flushed -/// batch hits the constraint and we drop the duplicate insert without losing the rest of -/// the batch). +/// batch hits the constraint; detaches the rejected row +/// and re-saves, so the duplicate is dropped without losing the rest of the batch). /// /// Implements the shared seam: is a /// best-effort, never-throwing entry point that simply Tells this actor and returns @@ -107,8 +107,8 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter Outcome = evt.Outcome.ToString(), }); } - db.SaveChanges(); - _log.Debug("AuditWriter flushed {Count} events", snapshot.Count); + var dropped = SaveDetachingRejectedRows(db); + _log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped); } catch (Exception ex) { @@ -116,6 +116,55 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter } } + /// + /// Saves the batch, detaching and retrying past any row the database rejects (R03:2.6). One + /// statement batch used to carry the whole flush: a single duplicate EventId (any at-least-once + /// producer — the audit seam is Tell-based through a singleton proxy that re-delivers across a + /// hand-over) threw for all 500 rows and the class remarks' per-row dedup claim was not what ran. + /// + /// The context holding the batch. + /// How many rows were dropped. + /// + /// Only a that NAMES its entries is handled here — that is the shape a + /// constraint violation takes (SQL 2601/2627 for the unique index). A failure with no entries (the + /// connection died, a deadlock) is the whole-batch failure it always was and propagates to the + /// caller's catch. Bounded by the batch size so a store that rejects every row still terminates. + /// + private int SaveDetachingRejectedRows(OtOpcUaConfigDbContext db) + { + var dropped = 0; + var attempts = 0; + var maxAttempts = db.ChangeTracker.Entries().Count() + 1; + while (true) + { + try + { + db.SaveChanges(); + return dropped; + } + catch (DbUpdateException ex) when (ex.Entries.Count > 0 && ++attempts < maxAttempts) + { + var duplicate = IsUniqueViolation(ex); + foreach (var entry in ex.Entries) + { + var eventId = (entry.Entity as ConfigAuditLog)?.EventId; + if (duplicate) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", eventId); + else + _log.Warning(ex, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", eventId); + entry.State = EntityState.Detached; + dropped++; + } + } + } + } + + /// SQL Server 2601 (unique index) / 2627 (unique constraint), the dedup contract's expected rejection. + /// The update failure. + /// for a duplicate-key rejection. + private static bool IsUniqueViolation(DbUpdateException ex) => + ex.InnerException is Microsoft.Data.SqlClient.SqlException { Number: 2601 or 2627 }; + /// protected override void PreRestart(Exception reason, object message) { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs new file mode 100644 index 00000000..bd64161a --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs @@ -0,0 +1,110 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.EntityFrameworkCore.Update; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:2.6 — one constraint hit used to drop the whole flush. The writer now detaches the rejected +/// row(s) and re-saves, so a duplicate EventId costs exactly that row. +/// +public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase +{ + [Fact] + public void A_rejected_row_is_dropped_and_the_rest_of_the_batch_is_persisted() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var duplicate = Guid.NewGuid(); + var rejecting = new RejectingDbFactory(dbName, root, new RejectOnce(duplicate)); + var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + + Watch(actor); + actor.Tell(PoisonPill.Instance); // PostStop flushes + ExpectTerminated(actor); + + using var db = rejecting.CreateDbContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(20, "every row but the rejected one lands"); + persisted.ShouldNotContain(duplicate); + // Positive control: the rows on either side of the rejected one are present. + persisted.ShouldContain(ids[6]); + persisted.ShouldContain(ids[8]); + } + + /// A failure that names no entry is a whole-batch failure, exactly as before. + [Fact] + public void A_connection_level_failure_still_drops_the_batch_without_looping() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var rejecting = new RejectingDbFactory(dbName, root, new AlwaysFailWithoutEntries()); + var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); + + for (var i = 0; i < 5; i++) actor.Tell(NewEvent(Guid.NewGuid())); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor); // terminates ⇒ the retry loop did not spin + + using var db = rejecting.CreateDbContext(); + db.ConfigAuditLogs.Count().ShouldBe(0); + } + + private static AuditEvent NewEvent(Guid eventId) => new() + { + EventId = eventId, Category = "Config", Action = "Edit", Actor = "joe", + OccurredAtUtc = DateTimeOffset.UtcNow, Outcome = AuditOutcome.Success, + DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(), + }; + + /// Throws a naming the entry for one EventId, the first time it is saved. + private sealed class RejectOnce(Guid eventId) : ISaveChangesInterceptor + { + private int _fired; + + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) + { + if (_fired == 1 || eventData.Context is null) return result; + var entry = eventData.Context.ChangeTracker.Entries() + .FirstOrDefault(e => e.Entity.EventId == eventId); + if (entry is null) return result; + _fired = 1; + // GetInfrastructure() hands back the pubternal InternalEntityEntry, which is the IUpdateEntry the + // real provider puts on the exception; the test only needs that one cast. +#pragma warning disable EF1001 + var updateEntry = (IUpdateEntry)entry.GetInfrastructure(); +#pragma warning restore EF1001 + throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601)", new[] { updateEntry }); + } + } + + private sealed class AlwaysFailWithoutEntries : ISaveChangesInterceptor + { + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) => + throw new DbUpdateException("connection lost (simulated)"); + } + + private sealed class RejectingDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor interceptor) + : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => new( + new DbContextOptionsBuilder() + .UseInMemoryDatabase(dbName, root) + .AddInterceptors(interceptor) + .Options); + } +} From 63714e430a75dadb1aa48c4a6106d6aec5165b48 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:34:39 -0400 Subject: [PATCH 12/27] fix(controlplane): evict retired mesh channels and refresh PSKs off the request path - MeshGrpcCommandDialer.RetainOnly (IMeshCommandDialer, default no-op): CentralCommunicationActor hands the dialer the current endpoint set on every contact refresh, so a renamed host or moved port releases its cached GrpcChannel and idle HTTP/2 connection instead of living for the process (R03:1.3). - IMeshPskProvider.RefreshAsync (default = the old InvalidateAll): MeshPskProvider pre-resolves every known cluster's key concurrently, bounded like the sync path, and swaps an immutable snapshot in atomically; a failed read keeps that cluster's previous key. The actor calls it from ContactsLoaded (after SetKnownClusters, off-actor via PipeTo) instead of InvalidateAll, so the first ack per cluster per ContactRefreshSeconds no longer blocks a gRPC thread on a 3 s-bounded store read or gets denied past the bound (R03:3.4). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../CentralCommunicationActor.cs | 57 +++++++- .../Communication/IMeshPskProvider.cs | 117 +++++++++++++++- .../Communication/MeshGrpcCommandDialer.cs | 33 +++++ .../CentralCommunicationActorTests.cs | 41 ++++++ .../MeshGrpcCommandDialerTests.cs | 29 ++++ .../MeshPskProviderRefreshTests.cs | 129 ++++++++++++++++++ 6 files changed, 404 insertions(+), 2 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs index cb1ea253..16f7b5d4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs @@ -201,6 +201,9 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers LoadContactsFromDb(); }); Receive(HandleContactsLoaded); + Receive(msg => _log.Debug("Mesh PSK refresh completed for {Count} cluster(s)", msg.ClusterCount)); + Receive(msg => _log.Warning(msg.Error, + "Mesh PSK refresh failed; previously resolved keys stay in service until the next refresh")); Receive(HandleCommandDispatched); // A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler @@ -710,6 +713,20 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers _targets = msg.Targets; + // R03:1.3 — release the channels of endpoints that left the dispatch set (host renamed, port + // moved). Only the resolved dialer is asked; nothing is resolved just to prune it. + if (_dialer is { } dialer) + { + try + { + dialer.RetainOnly(_targets.Select(t => t.Endpoint).ToHashSet(StringComparer.Ordinal)); + } + catch (Exception ex) + { + _log.Warning(ex, "Mesh dialer failed to release channels for retired endpoints; they stay cached until the next refresh"); + } + } + if (_targets.Count == 0) { _log.Warning( @@ -780,7 +797,37 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers } pskProvider.SetKnownClusters(msg.ClusterIds); - pskProvider.InvalidateAll(); + + // R03:3.4 — refresh the per-cluster keys OFF the request path and swap them in atomically, + // instead of dropping the cache and making the first ack per cluster re-read the store on a + // gRPC thread. A provider that does not pre-resolve (the interface default, and every fake) + // still drops its cache here, synchronously, exactly as before. + CancellationToken ct; + try + { + ct = _lifecycle.Token; + } + catch (ObjectDisposedException) + { + return; // Stopping. + } + var self = Self; + Task refresh; + try + { + refresh = pskProvider.RefreshAsync(msg.ClusterIds, ct); + } + catch (Exception ex) + { + _log.Warning(ex, "Mesh PSK refresh threw synchronously; cached keys are unchanged until the next refresh"); + refresh = Task.CompletedTask; + } + if (!refresh.IsCompletedSuccessfully) + { + refresh.PipeTo(self, + success: () => new PskRefreshed(msg.ClusterIds.Count), + failure: ex => new PskRefreshFailed(ex)); + } if (!_gatePublished) { @@ -795,6 +842,14 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// Internal tick asking for a contact-point refresh. public sealed record RefreshContacts; + /// The asynchronous PSK refresh (R03:3.4) completed. + /// How many clusters were refreshed. + public sealed record PskRefreshed(int ClusterCount); + + /// The asynchronous PSK refresh (R03:3.4) threw; the previous keys stay in service. + /// The failure. + public sealed record PskRefreshFailed(Exception Error); + /// Result of a contact-point load, piped back from the DB read. /// /// The gRPC dispatch set: one entry per enabled, non-maintenance row that carries a diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs index a7282694..bdab7ba7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs @@ -106,6 +106,26 @@ public interface IMeshPskProvider /// Drops every cached key. void InvalidateAll(); + + /// + /// Re-reads the keys for from the store OFF the request path and + /// swaps them into the cache atomically (R03:3.4). Until the refresh completes, lookups keep + /// serving the previous key; a cluster whose read fails keeps its previous key too. Replaces + /// "invalidate on every refresh", which made the first inbound ack per cluster per + /// ContactRefreshSeconds block a gRPC thread on a synchronous, 3 s-bounded store read. + /// + /// The clusters central currently knows (the same set given to ). + /// Cancels the refresh. + /// Completes when the swap has happened. + /// + /// The default implementation is the pre-R03:3.4 behaviour — drop the cache and let the next lookup + /// re-read — so a fake or a non-caching provider needs nothing more. + /// + Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) + { + InvalidateAll(); + return Task.CompletedTask; + } } /// @@ -144,6 +164,16 @@ public sealed class MeshPskProvider : IMeshPskProvider private readonly ConcurrentDictionary> _cache = new(StringComparer.OrdinalIgnoreCase); + /// + /// Keys pre-resolved by , consulted after the config map and before the + /// lazy . Swapped wholesale (volatile reference to an immutable dictionary), + /// never mutated, so a lookup racing a refresh sees the whole previous snapshot or the whole next + /// one. A cluster missing from it — never refreshed, or whose read failed — falls through to the + /// lazy path exactly as before, which is what keeps "refresh failed" from meaning "no key". + /// + private volatile IReadOnlyDictionary _refreshed = + new Dictionary(StringComparer.OrdinalIgnoreCase); + /// /// The known-cluster gate's membership set, or when the gate is /// inactive (never published, or published empty — see ). @@ -210,6 +240,13 @@ public sealed class MeshPskProvider : IMeshPskProvider return true; } + // Pre-resolved by the asynchronous refresh: no store read on this (gRPC request) thread at all. + if (_refreshed.TryGetValue(clusterId, out var refreshed)) + { + psk = refreshed; + return true; + } + var entry = _cache.GetOrAdd( clusterId, static (id, self) => new Lazy( @@ -288,7 +325,85 @@ public sealed class MeshPskProvider : IMeshPskProvider } /// - public void InvalidateAll() => _cache.Clear(); + public void InvalidateAll() + { + _cache.Clear(); + _refreshed = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + /// + public async Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(clusterIds); + + var previous = _refreshed; + var next = new Dictionary(StringComparer.OrdinalIgnoreCase); + var ids = clusterIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + + // Resolve every cluster concurrently — each read bounded like the synchronous path — and carry a + // failed cluster's PREVIOUS refreshed key forward: a transient store fault must not turn into a + // window in which that cluster has no key at all. + var reads = ids.Select(async id => + { + var value = await ResolveFromStoreAsync(id, ct).ConfigureAwait(false); + return (Id: id, Value: value); + }).ToList(); + foreach (var (id, value) in await Task.WhenAll(reads).ConfigureAwait(false)) + { + if (!string.IsNullOrWhiteSpace(value)) next[id] = value; + else if (previous.TryGetValue(id, out var kept)) next[id] = kept; + } + + // The swap. Then drop the lazy entries for the clusters now covered, so a key rotated between + // refreshes is served from the new snapshot rather than from a stale lazy resolution; entries + // for clusters NOT covered (their read failed and they had no previous value) are left alone. + _refreshed = next; + foreach (var id in next.Keys) _cache.TryRemove(id, out _); + } + + /// + /// The asynchronous twin of : same name derivation, same bound, same + /// "anything but a usable key is null" contract — awaited instead of blocked on, because this one + /// runs on a refresh rather than inside a gRPC interceptor. + /// + /// The application Cluster id. + /// The caller's token, linked under the store-read bound. + /// The stored key, or null. + private async Task ResolveFromStoreAsync(string clusterId, CancellationToken ct) + { + SecretName name; + try + { + name = new SecretName(SecretNamePrefix + clusterId); + } + catch (ArgumentException ex) + { + _logger.LogWarning( + ex, + "Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be " + + "refreshed from the store. Supply it through MeshTransport:ClusterPsks instead, or " + + "rename the cluster to [a-z0-9._/-].", + clusterId); + return null; + } + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + cts.CancelAfter(StoreReadTimeout); + try + { + return await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Failed to refresh the mesh PSK for cluster {ClusterId} from secret {SecretName}; the " + + "previously resolved key (if any) stays in service until the next refresh.", + clusterId, + name.Value); + return null; + } + } /// /// Reads OT-MESH-PSK-{clusterId} from the Secrets store, converting every failure mode diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs index a10d228a..31a19b13 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs @@ -34,6 +34,15 @@ public interface IMeshCommandDialer /// rather than dialled unauthenticated. /// Task DispatchAsync(string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct); + + /// + /// Releases any per-endpoint resources for endpoints NOT in + /// (R03:1.3). Central calls it on every contact refresh with the current dispatch set, so a + /// renamed host or moved port does not leave the old channel and its idle HTTP/2 connection + /// alive for the life of the process. Default no-op for dialers that cache nothing. + /// + /// The endpoints still in the dispatch set. + void RetainOnly(IReadOnlyCollection nodeEndpoints) { } } /// @@ -181,6 +190,30 @@ public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable await call.ResponseAsync.ConfigureAwait(false); } + /// + public void RetainOnly(IReadOnlyCollection nodeEndpoints) + { + ArgumentNullException.ThrowIfNull(nodeEndpoints); + if (_disposed) return; + + var keep = nodeEndpoints as IReadOnlySet ?? nodeEndpoints.ToHashSet(StringComparer.Ordinal); + foreach (var endpoint in _channels.Keys) + { + if (keep.Contains(endpoint)) continue; + // A dial that already took this channel out of the cache before the removal fails once, as + // a named per-node RpcException the actor logs; the next dial to that endpoint (if it ever + // returns to the set) builds a fresh channel. Bounded to the dials in flight at the instant + // of a contact refresh, the same accepted race Dispose documents. + if (_channels.TryRemove(endpoint, out var channel)) + { + channel.Dispose(); + _log.LogInformation( + "Mesh dialer released the cached channel for {Endpoint}: it is no longer in the dispatch set", + endpoint); + } + } + } + /// Marks the dialer disposed and disposes every cached gRPC channel. public void Dispose() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs index 9843c8fe..51dcd046 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs @@ -930,6 +930,34 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase /// A spawned gRPC-mode actor plus the two seams the tests observe it through. private sealed record GrpcRig(IActorRef Actor, RecordingDialer Dialer, RecordingPskProvider Psk); + /// + /// R03:1.3 — every contact refresh hands the dialer the CURRENT endpoint set so it can release the + /// channels of endpoints that left it. + /// + [Fact] + public void Each_contact_refresh_tells_the_dialer_which_endpoints_remain() + { + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false), ("host-b:4053", "host-b", 4056, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + // The dialer is resolved lazily on first use; a dispatch forces that, and the NEXT refresh prunes. + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5)); + + using (var db = factory.CreateDbContext()) + { + db.ClusterNodes.Single(n => n.NodeId == "host-b:4053").MaintenanceMode = true; + db.SaveChanges(); + } + rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts()); + + AwaitCondition( + () => rig.Dialer.RetainCalls.Any(set => set.Count == 1 && set.Single().Contains("host-a")), + TimeSpan.FromSeconds(5)); + rig.Dialer.RetainCalls.Last().ShouldBe(new[] { "http://host-a:4056" }); + } + private GrpcRig SpawnGrpcRig( IDbContextFactory factory, RecordingDialer? dialer = null, @@ -1014,6 +1042,19 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase : Task.CompletedTask; } + /// Every endpoint set handed to , in order. + public IReadOnlyList> RetainCalls + { + get { lock (_gate) return _retainCalls.ToList(); } + } + + private readonly List> _retainCalls = []; + + public void RetainOnly(IReadOnlyCollection nodeEndpoints) + { + lock (_gate) _retainCalls.Add(nodeEndpoints.ToList()); + } + /// One recorded dial. public sealed record Dial(string Endpoint, CommandEnvelope Envelope, string ClusterId); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs index 2d110eb6..fc177361 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs @@ -224,6 +224,35 @@ public sealed class MeshGrpcCommandDialerTests Should.NotThrow(dialer.Dispose); } + /// + /// R03:1.3 — a channel for an endpoint that left the dispatch set (host renamed, port moved) is + /// released on the next refresh instead of living, with its idle connection, for the life of central. + /// Positive control: a dial to that endpoint afterwards rebuilds a channel. + /// + [Fact] + public async Task RetainOnly_releases_channels_for_endpoints_that_left_the_dispatch_set() + { + var dialer = new MeshGrpcCommandDialer( + new FakePskProvider(("SITE-A", "psk-site-a")), + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }), + NullLogger.Instance); + foreach (var endpoint in new[] { "http://127.0.0.1:1", "http://127.0.0.1:2" }) + { + await Should.ThrowAsync( + () => dialer.DispatchAsync(endpoint, Envelope(), "SITE-A", CancellationToken.None)); + } + dialer.CachedChannelCount.ShouldBe(2); + + dialer.RetainOnly(new[] { "http://127.0.0.1:1" }); + + dialer.CachedChannelCount.ShouldBe(1); + + await Should.ThrowAsync( + () => dialer.DispatchAsync("http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None)); + dialer.CachedChannelCount.ShouldBe(2, "a returning endpoint gets a fresh channel"); + Should.NotThrow(dialer.Dispose); + } + [Fact] public void It_activates_from_the_container_despite_the_optional_client_factory_parameter() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs new file mode 100644 index 00000000..e727c153 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs @@ -0,0 +1,129 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; +using ZB.MOM.WW.Secrets.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; + +/// +/// R03:3.4 — central invalidated every cached key on every contact refresh (default 60 s), so the +/// first inbound ack per cluster per minute blocked a gRPC thread on a synchronous, 3 s-bounded store +/// read and was denied past the bound. pre-resolves the +/// keys off the request path and swaps them in atomically. +/// +public sealed class MeshPskProviderRefreshTests +{ + private static MeshPskProvider Provider(ISecretResolver resolver) => + new(Options.Create(new MeshTransportOptions()), resolver, NullLogger.Instance); + + [Fact] + public async Task After_a_refresh_the_lookup_is_served_without_touching_the_store() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + resolver.Calls.ShouldBe(1); + + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-1"); + resolver.Calls.ShouldBe(1, "the request path did not consult the store"); + provider.Matches("SITE-A", "k-1").ShouldBeTrue(); + } + + /// + /// While a refresh is in flight the PREVIOUS key is still served (no invalidate-then-reread window); + /// once it completes the rotated key is served. Positive control on both sides of the swap. + /// + [Fact] + public async Task A_rotated_key_is_swapped_in_atomically_with_the_old_key_served_until_then() + { + var resolver = new GatedResolver { Value = "old" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + provider.TryGetPsk("SITE-A", out var before).ShouldBeTrue(); + before.ShouldBe("old"); + + resolver.Value = "new"; + resolver.Hold(); + var refresh = provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + resolver.Entered.Wait(TimeSpan.FromSeconds(5)).ShouldBeTrue(); + + // Mid-refresh: the old key, from memory — the store is NOT re-read on this path. + var callsMidRefresh = resolver.Calls; + provider.TryGetPsk("SITE-A", out var during).ShouldBeTrue(); + during.ShouldBe("old"); + resolver.Calls.ShouldBe(callsMidRefresh); + + resolver.Release(); + await refresh; + provider.TryGetPsk("SITE-A", out var after).ShouldBeTrue(); + after.ShouldBe("new"); + } + + [Fact] + public async Task A_cluster_whose_refresh_fails_keeps_its_previous_key() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + resolver.Throw = new InvalidOperationException("store offline"); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-1"); + } + + /// A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour). + [Fact] + public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily() + { + var resolver = new GatedResolver { Value = "k-b" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-B", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-b"); + resolver.Calls.ShouldBe(2); + } + + [Fact] + public async Task InvalidateAll_drops_refreshed_keys_too() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.InvalidateAll(); + resolver.Value = "k-2"; + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-2"); + } + + private sealed class GatedResolver : ISecretResolver + { + private int _calls; + private volatile TaskCompletionSource? _gate; + + public string? Value { get; set; } + public Exception? Throw { get; set; } + public int Calls => Volatile.Read(ref _calls); + public ManualResetEventSlim Entered { get; } = new(); + + public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _gate?.TrySetResult(); + + public async Task GetAsync(SecretName name, CancellationToken ct) + { + Interlocked.Increment(ref _calls); + Entered.Set(); + if (_gate is { } gate) await gate.Task.WaitAsync(ct); + if (Throw is not null) throw Throw; + return Value; + } + } +} From e04156d8f7006067518955358281566f199939e1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:39:12 -0400 Subject: [PATCH 13/27] fix(host): refuse unknown API-key ids in memory and budget store misses; document the mesh gate's boot window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BoundedApiKeyStore's negative cache was keyed by key id, so a flood of DISTINCT well-formed otua__x credentials missed it every time and cost one SQL query per attempt under the SDK's SessionManager-wide lock, no credential needed (R03:3.3). - IApiKeyStore.ListKeyIdsAsync (default null = unsupported; ConfigDbApiKeyStore lists every ApiKeyId): the bounded store keeps a known-id set refreshed every StoreCacheSeconds (single-flight, failed refresh keeps the previous set) and refuses an unknown id without a store read ({outcome=unknown_id}). - A process-wide miss budget (token bucket, Security:ApiKeys:MissBurst=100 / MissRefillPerSecond=20, ApiKeyStoreGuardOptions) bounds lookups that reach the store and miss while no set is available; a hit refunds its token. Per-endpoint accounting needs the OpcUaServer impersonation seam to carry the client address — follow-up. - docs/security.md: the new knobs, and the known-cluster gate's documented fail-open window at central boot (R03:4.2) plus the async PSK refresh (R03:3.4). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/security.md | 44 +++++ .../DataPlaneAuthRegistration.cs | 5 +- .../OpcUa/ApiKeyStoreGuardOptions.cs | 56 ++++++ .../OpcUa/BoundedApiKeyStore.cs | 117 +++++++++++- .../OpcUa/ConfigDbApiKeyStore.cs | 10 + .../OpcUa/IApiKeyStore.cs | 10 + .../OpcUa/BoundedApiKeyStoreKnownIdTests.cs | 173 ++++++++++++++++++ .../OpcUa/BoundedApiKeyStoreTests.cs | 11 +- 8 files changed, 421 insertions(+), 5 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs diff --git a/docs/security.md b/docs/security.md index 87ac8d22..6e44569a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -245,6 +245,25 @@ with two `Security:ApiKeys` knobs: a completed lookup (a hit or an authoritative miss) is stored, and a cached hit whose `ExpiresUtc` has since passed is re-read rather than served, so the TTL can never resurrect an expired key. +#### Pre-authentication guards (`KnownIdSet` / `MissBurst` / `MissRefillPerSecond`) + +The negative cache above is keyed by **key id**, so a flood of *distinct*, well-formed `otua__x` +credentials missed it every time and cost one indexed SQL query per attempt — under the SDK's +`SessionManager`-wide lock, with no credential required (review 03 §3.3). `BoundedApiKeyStore` now puts two +guards in front of the store, bound from the same `Security:ApiKeys` section +(`ApiKeyStoreGuardOptions`, Host-owned): + +| Key | Default | Meaning | +|---|---|---| +| `KnownIdSet` | `true` | Keep an in-memory set of every `ApiKeyId` the ConfigDb holds, refreshed every `StoreCacheSeconds` (single-flight; a failed refresh keeps the previous set). An id outside it is refused in memory — `otopcua.opcua.apikey_store.bounded{outcome=unknown_id}` — and the store is never asked. A key minted less than `StoreCacheSeconds` ago is refused until the next refresh (the same window `Enabled` revocation already has). `StoreCacheSeconds = 0` disables the set with the cache. | +| `MissBurst` | `100` | Token bucket on lookups that **reach** the store and **miss**, for when the set is unavailable (not loaded yet, or the store cannot list ids). Exhausted ⇒ unknown ids are refused without a store read (`{outcome=miss_budget}`, a Warning-free deny through the authenticator's catch-all). A hit refunds its token, so legitimate clients are never charged. `0` disables the budget. | +| `MissRefillPerSecond` | `20` | Refill rate of the miss bucket. | + +The budget is **process-wide, not per remote endpoint**: the OPC UA SDK's impersonation callback (the +`IOpcUaUserAuthenticator` seam) does not carry the client's address, so per-endpoint accounting needs that +seam widened in the OpcUaServer project (a Wave 3 follow-up). Neither guard applies to a driver-only node: +its `ArtifactApiKeyStore` is two in-memory dictionary probes and is never wrapped. + ### LDAP bind flow (`OtOpcUaLdapAuthService`) LDAP is configured under the `Security:Ldap` section (bound to `LdapOptions`, `src/Server/ZB.MOM.WW.OtOpcUa.Security/Ldap/LdapOptions.cs`, `SectionName = "Security:Ldap"`). The app authenticator is `OtOpcUaLdapAuthService` — a thin wrapper around the shared `ZB.MOM.WW.Auth.Ldap` directory client that adds two app-only concerns the shared library deliberately does not model: the `Enabled` master switch and `DevStubMode`. Since #520 it serves the **Admin UI cookie login (`/auth/login`) only** — `LdapOpcUaUserAuthenticator` is deleted and the OPC UA UserName path authenticates API keys instead (see [API-key identity](#api-key-identity-opc-ua-data-plane)). @@ -1116,6 +1135,31 @@ See `archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md` (R3-01) for the fu record, and `docs/Configuration.md` §`MeshTransport` / §`ConfigSource`-`ConfigServe` and `docs/Telemetry.md` for the affected listener ports. +### The known-cluster gate is fail-OPEN until central's first ClusterNode load (review 03 §4.2) + +`CentralCommandAuthInterceptor` verifies an inbound deploy ack against the PSK of the cluster named in +`x-otopcua-cluster`. `MeshPskProvider` consults a **known-cluster gate** first — the set of cluster ids +central read from `ClusterNode` on its last contact refresh — and refuses anything outside it in O(1), +touching neither the config map nor the secret store. That gate is **inactive until the first successful +load** (`CentralCommunicationActor` publishes it from `ContactsLoaded`; an empty publication also +deactivates it, deliberately, so "the DB told me nothing" never becomes a fleet-wide authentication +outage). During a central boot with the ConfigDb down, therefore: + +- an unauthenticated caller can name *any* `x-otopcua-cluster` value and make central perform one bounded + (3 s) secret-store read plus one Warning line per request — the cost/oracle the gate exists to remove; +- **keys are still required**: nothing is admitted without the matching PSK, so this is a cost exposure, + not an authentication bypass; +- the window is bounded by the bootstrap retry (15 × 2 s) and the `ContactRefreshSeconds` (60 s) periodic + refresh, and closes on the first load that succeeds. The actor logs the condition on every failed load + (*"The known-cluster gate was NOT refreshed either … it is INACTIVE"*). + +Since review 03 §3.4 the per-cluster keys are **refreshed asynchronously** from the same handler +(`IMeshPskProvider.RefreshAsync`: concurrent bounded reads, atomic snapshot swap, a failed read keeps the +previous key) instead of being invalidated on every refresh, so the first ack per cluster per minute no +longer pays a synchronous store read on a gRPC thread. Accepted as documented, bounded behaviour; the +alternative — refusing every cluster until SQL answers — would turn a ConfigDb outage into a deploy-ack +outage on top of it. + --- ## OTOPCUA0001 Analyzer — Compile-Time Guard diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs index e7c301e4..eace8505 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs @@ -80,7 +80,10 @@ public static class DataPlaneAuthRegistration sp.GetRequiredService>().Value)), sp.GetRequiredService>(), sp.GetRequiredService().CreateLogger(), - sp.GetRequiredService())); + sp.GetRequiredService(), + // R03:3.3 — the known-id set + miss budget in front of the SQL lookup; same section as + // ApiKeyOptions, Host-owned knobs. + ApiKeyStoreGuardOptions.Bind(sp.GetService()))); services.AddSingleton(); // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs new file mode 100644 index 00000000..3103b982 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs @@ -0,0 +1,56 @@ +using Microsoft.Extensions.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// +/// The pre-authentication guards puts in front of the ConfigDb key +/// lookup (R03:3.3). Bound from the same Security:ApiKeys section as ApiKeyOptions but +/// owned by the Host, where the store lives. +/// +/// +/// +/// Why. The negative cache was keyed by key id, so a flood of DISTINCT well-formed +/// otua_<random>_x credentials missed it every time and cost one indexed SQL query per +/// attempt — under the SDK's SessionManager-wide lock — with no credential needed. Two guards close +/// that: a known-id set (every ApiKeyId the ConfigDb holds, refreshed every +/// StoreCacheSeconds) rejects an unknown id in memory, and a miss budget (token +/// bucket) bounds how many lookups that reach the store and MISS are allowed per second when the +/// set is unavailable. +/// +/// +/// The budget is process-wide, not per remote endpoint: the OPC UA SDK's impersonation callback +/// (the IOpcUaUserAuthenticator seam) does not carry the client's address. Per-endpoint +/// accounting needs that seam widened in the OpcUaServer project — a follow-up, not this lane. +/// +/// +public sealed class ApiKeyStoreGuardOptions +{ + /// The configuration section the knobs live in (shared with ApiKeyOptions). + public const string SectionName = "Security:ApiKeys"; + + /// + /// Whether the known-id set is consulted. restores a store read per unknown + /// id. Only meaningful when the inner store can list ids (the ConfigDb store can; the artifact + /// store is never wrapped). + /// + public bool KnownIdSet { get; set; } = true; + + /// + /// How many store MISSES (lookups that reach the inner store and find no key) may burst before + /// further lookups for unknown ids are refused without a store read. 0 disables the budget. + /// + public int MissBurst { get; set; } = 100; + + /// How many miss tokens are restored per second, up to . + public double MissRefillPerSecond { get; set; } = 20; + + /// Binds the section, with defaults for anything absent. + /// The app configuration; null yields the defaults. + /// The bound options. + public static ApiKeyStoreGuardOptions Bind(IConfiguration? configuration) + { + var options = new ApiKeyStoreGuardOptions(); + configuration?.GetSection(SectionName).Bind(options); + return options; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index af0fca3f..bb3154ed 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -56,6 +56,21 @@ public sealed class BoundedApiKeyStore : IApiKeyStore private readonly ConcurrentDictionary> _keys = new(StringComparer.Ordinal); private readonly ConcurrentDictionary>> _grants = new(StringComparer.Ordinal); + private readonly ApiKeyStoreGuardOptions _guard; + + // R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds + // (single-flight; a refresh that fails or times out keeps the previous set). Null until the first + // successful load, or when the inner store cannot list ids — then the per-id path runs as before. + private volatile IReadOnlySet? _knownIds; + private DateTimeOffset _knownIdsStampedUtc = DateTimeOffset.MinValue; + private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1); + + // R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards + // the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the + // configured rate instead of one SQL round trip per attempt under the SessionManager lock. + private readonly object _bucketGate = new(); + private double _missTokens; + private DateTimeOffset _missRefilledUtc; /// Initializes a new instance of the class. /// The store being bounded. @@ -66,11 +81,13 @@ public sealed class BoundedApiKeyStore : IApiKeyStore /// DataPlaneAuthRegistration for why a second clock here would be a hazard rather than a /// detail. /// + /// The pre-authentication guards (R03:3.3); defaults when omitted. public BoundedApiKeyStore( IApiKeyStore inner, IOptions options, ILogger log, - TimeProvider time) + TimeProvider time, + ApiKeyStoreGuardOptions? guard = null) { ArgumentNullException.ThrowIfNull(inner); ArgumentNullException.ThrowIfNull(options); @@ -81,8 +98,14 @@ public sealed class BoundedApiKeyStore : IApiKeyStore _options = options; _log = log; _time = time; + _guard = guard ?? new ApiKeyStoreGuardOptions(); + _missTokens = Math.Max(0, _guard.MissBurst); + _missRefilledUtc = time.GetUtcNow(); } + /// The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards. + public IReadOnlySet? KnownKeyIds => _knownIds; + /// The store being bounded. /// /// Exposed so the composition root's wiring can be asserted. A decorator registered in one branch @@ -115,12 +138,104 @@ public sealed class BoundedApiKeyStore : IApiKeyStore return cached; } + // R03:3.3, guard 1 — an id the ConfigDb does not hold is refused in memory; the store is never asked. + var known = await KnownIdsAsync(ttl, ct).ConfigureAwait(false); + if (known is not null && !known.Contains(apiKeyId)) + { + OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair("outcome", "unknown_id")); + return null; + } + + // R03:3.3, guard 2 — a lookup that will reach the store needs a miss token unless the id is known + // to exist (a known id is a real key being verified, never a guess). + if (known is null && !TryTakeMissToken()) + { + OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair("outcome", "miss_budget")); + throw new InvalidOperationException( + "ApiKeyStore: the unknown-key miss budget is exhausted (Security:ApiKeys:MissBurst / " + + "MissRefillPerSecond); refusing this lookup without a store read."); + } + var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct) .ConfigureAwait(false); + if (fresh is not null && known is null) RefundMissToken(); if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh); return fresh; } + /// + /// The known-id set, refreshed when older than . Single-flight: concurrent + /// callers during a refresh use the previous set (or, before the first load, the per-id path). A + /// refresh that fails or exceeds the deadline logs and keeps the previous set. + /// + /// The refresh interval (the cache TTL); non-positive disables the set. + /// The caller's token. + /// The set, or null. + private async Task?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct) + { + if (!_guard.KnownIdSet || ttl <= TimeSpan.Zero) return null; + + var now = _time.GetUtcNow(); + var current = _knownIds; + if (current is not null && now - _knownIdsStampedUtc < ttl) return current; + + if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current; + try + { + // Re-check under the gate: another caller may have refreshed while this one waited. + if (_knownIds is { } refreshed && now - _knownIdsStampedUtc < ttl) return refreshed; + var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false); + if (ids is null) + { + // Unsupported by the inner store: say so once and stop asking. + _guard.KnownIdSet = false; + return null; + } + _knownIds = ids; + _knownIdsStampedUtc = _time.GetUtcNow(); + return ids; + } + catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) + { + _log.LogWarning(ex, + "ApiKeyStore: refreshing the known key-id set failed; {Mode}", + current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force"); + return current; + } + finally + { + _knownIdsRefresh.Release(); + } + } + + /// Takes one miss token, refilling by elapsed time first; always succeeds when the budget is disabled. + /// when the budget is exhausted. + private bool TryTakeMissToken() + { + var burst = _guard.MissBurst; + if (burst <= 0) return true; + lock (_bucketGate) + { + var now = _time.GetUtcNow(); + var elapsed = (now - _missRefilledUtc).TotalSeconds; + if (elapsed > 0) + { + _missTokens = Math.Min(burst, _missTokens + elapsed * Math.Max(0, _guard.MissRefillPerSecond)); + _missRefilledUtc = now; + } + if (_missTokens < 1) return false; + _missTokens -= 1; + return true; + } + } + + /// A lookup that reached the store and HIT was not a miss: its token goes back. + private void RefundMissToken() + { + if (_guard.MissBurst <= 0) return; + lock (_bucketGate) _missTokens = Math.Min(_guard.MissBurst, _missTokens + 1); + } + // Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect. /// public async Task> FindGrantsAsync(string principal, CancellationToken ct) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs index c3f9c1f5..52fcbe51 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs @@ -125,6 +125,16 @@ public sealed class ConfigDbApiKeyStore : IApiKeyStore return await LoadGrantsAsync(db, principal, ct).ConfigureAwait(false); } + /// + public async Task?> ListKeyIdsAsync(CancellationToken ct) + { + await using var db = await _factory.CreateDbContextAsync(ct).ConfigureAwait(false); + // One projection over the whole (small) table — the unique index on ApiKeyId makes this an index + // scan — so an unknown id can be refused in memory instead of costing a query per attempt. + var ids = await db.ApiKeys.AsNoTracking().Select(k => k.ApiKeyId).ToListAsync(ct).ConfigureAwait(false); + return ids.ToHashSet(StringComparer.Ordinal); + } + /// Loads a principal's grants, confined to this node's cluster. /// /// The full grant set, not just the Cluster-scoped rows. The old diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs index c2f82dae..93057153 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs @@ -35,6 +35,16 @@ public interface IApiKeyStore /// The cancellation token. /// The matching ACL rows; empty when none. Task> FindGrantsAsync(string principal, CancellationToken ct); + + /// + /// Every ApiKeyId the store holds (enabled or not), for the pre-authentication known-id set + /// (R03:3.3); when the store cannot cheaply list them, in which case callers + /// fall back to a per-id lookup. The set only answers "does this id exist at all" — enabled, expiry + /// and the secret are still decided by . + /// + /// The cancellation token. + /// The id set, or null when unsupported. + Task?> ListKeyIdsAsync(CancellationToken ct) => Task.FromResult?>(null); } /// An API key resolved for authentication, with the ACL grants naming its principal. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs new file mode 100644 index 00000000..7051b151 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs @@ -0,0 +1,173 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// R03:3.3 — the negative cache was keyed by key id, so a flood of DISTINCT well-formed ids cost one +/// SQL query per attempt under the SessionManager lock, no credential needed. Two guards: +/// the known-id set (unknown ids refused in memory) and the miss budget (a token bucket on lookups +/// that reach the store and miss, for when the set is unavailable). +/// +public sealed class BoundedApiKeyStoreKnownIdTests +{ + private static readonly DateTimeOffset T0 = new(2026, 10, 22, 12, 0, 0, TimeSpan.Zero); + + [Fact] + public async Task An_unknown_id_is_refused_without_reaching_the_store_and_a_known_one_still_does() + { + var inner = new ListingStore("k-real"); + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + (await sut.FindAsync("k-guess-1", T0.UtcDateTime, default)).ShouldBeNull(); + (await sut.FindAsync("k-guess-2", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(0, "unknown ids never reach the store"); + inner.ListCalls.ShouldBe(1, "one set load serves every unknown id inside the TTL"); + + // Positive control: the real id reaches the store and is found. + (await sut.FindAsync("k-real", T0.UtcDateTime, default)).ShouldNotBeNull(); + inner.FindCalls.ShouldBe(1); + } + + [Fact] + public async Task The_set_is_refreshed_after_the_TTL_so_a_newly_minted_key_becomes_visible() + { + var inner = new ListingStore("k-old"); + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + (await sut.FindAsync("k-new", T0.UtcDateTime, default)).ShouldBeNull(); + inner.Mint("k-new"); + // Still inside the TTL: the stale set refuses it (the documented window). + clock.Advance(TimeSpan.FromSeconds(3)); + (await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(0); + + clock.Advance(TimeSpan.FromSeconds(3)); + (await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + inner.ListCalls.ShouldBe(2); + } + + [Fact] + public async Task A_failed_set_load_falls_back_to_the_per_id_lookup_and_keeps_an_older_set() + { + var inner = new ListingStore("k-real") { ListThrows = true }; + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + // No set yet and the load fails: the per-id path runs (old behaviour), nothing is wedged. + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(1); + sut.KnownKeyIds.ShouldBeNull(); + + inner.ListThrows = false; + clock.Advance(TimeSpan.FromSeconds(6)); + (await sut.FindAsync("k-guess-2", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + sut.KnownKeyIds.ShouldNotBeNull(); + inner.FindCalls.ShouldBe(1, "once the set loaded, the guess was refused in memory"); + + // A later failed refresh keeps the previous set in force. + inner.ListThrows = true; + clock.Advance(TimeSpan.FromSeconds(6)); + (await sut.FindAsync("k-guess-3", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(1); + } + + /// With no id set available, misses are rate-limited; hits are not charged; time refills. + [Fact] + public async Task The_miss_budget_bounds_store_misses_when_no_set_is_available() + { + var inner = new ListingStore("k-real") { SupportsListing = false }; + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 0, missBurst: 3, refillPerSecond: 1); + + for (var i = 0; i < 3; i++) + (await sut.FindAsync($"k-guess-{i}", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(3); + + // Budget exhausted: refused without a store read, as a throw the authenticator denies on. + await Should.ThrowAsync(() => sut.FindAsync("k-guess-9", T0.UtcDateTime, default)); + inner.FindCalls.ShouldBe(3); + + // Refill: one second restores one token. + clock.Advance(TimeSpan.FromSeconds(1)); + (await sut.FindAsync("k-guess-10", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(4); + + // A hit is not a miss: it reaches the store even with the bucket empty? No — it needs a token to + // be attempted, but the token is refunded, so the next attempt still has it. Prove the refund. + clock.Advance(TimeSpan.FromSeconds(1)); + (await sut.FindAsync("k-real", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + (await sut.FindAsync("k-guess-11", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(6); + } + + [Fact] + public async Task The_guards_are_off_when_the_cache_is_off_restoring_a_read_per_lookup() + { + var inner = new ListingStore("k-real"); + var sut = Bounded(inner, new FakeClock(T0), cacheSeconds: 0, missBurst: 0); + + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(2); + inner.ListCalls.ShouldBe(0); + } + + private static BoundedApiKeyStore Bounded( + IApiKeyStore inner, TimeProvider clock, int cacheSeconds, int missBurst = 100, double refillPerSecond = 20) => new( + inner, + Options.Create(new ApiKeyOptions + { + Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = cacheSeconds, + }), + NullLogger.Instance, + clock, + new ApiKeyStoreGuardOptions { MissBurst = missBurst, MissRefillPerSecond = refillPerSecond }); + + private sealed class ListingStore(params string[] ids) : IApiKeyStore + { + private readonly HashSet _ids = ids.ToHashSet(StringComparer.Ordinal); + private int _findCalls; + private int _listCalls; + + public bool SupportsListing { get; set; } = true; + public bool ListThrows { get; set; } + public int FindCalls => _findCalls; + public int ListCalls => _listCalls; + + public void Mint(string id) => _ids.Add(id); + + public Task FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct) + { + Interlocked.Increment(ref _findCalls); + return Task.FromResult(_ids.Contains(apiKeyId) + ? new ApiKeyLookup(new ApiKey { ApiKeyId = apiKeyId, Name = "k", SecretHash = "h", Principal = "p" }, []) + : null); + } + + public Task> FindGrantsAsync(string principal, CancellationToken ct) => + Task.FromResult>([]); + + public Task?> ListKeyIdsAsync(CancellationToken ct) + { + if (!SupportsListing) return Task.FromResult?>(null); + Interlocked.Increment(ref _listCalls); + if (ListThrows) throw new InvalidOperationException("ConfigDb unreachable (simulated)"); + return Task.FromResult?>(_ids.ToHashSet(StringComparer.Ordinal)); + } + } + + private sealed class FakeClock(DateTimeOffset utcNow) : TimeProvider + { + private DateTimeOffset _now = utcNow; + public override DateTimeOffset GetUtcNow() => _now; + public void Advance(TimeSpan by) => _now = _now.Add(by); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs index e8081db1..65201905 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs @@ -363,7 +363,10 @@ public sealed class BoundedApiKeyStoreTests { var (_, inner) = SetUp(); var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); - var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock); + // The miss budget (R03:3.3) is OFF here on purpose: this fact floods 1025 distinct unknown ids — the + // exact shape the budget refuses — to pin the cache CEILING, which is a separate rule. + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock, + guard: new ApiKeyStoreGuardOptions { MissBurst = 0 }); // Seed the real key, then flood past the ceiling with ids that do not exist. (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); @@ -477,7 +480,8 @@ public sealed class BoundedApiKeyStoreTests int timeoutMs, int cacheSeconds, TimeProvider? clock = null, - ILogger? log = null) => new( + ILogger? log = null, + ApiKeyStoreGuardOptions? guard = null) => new( inner, Options.Create(new ApiKeyOptions { @@ -487,7 +491,8 @@ public sealed class BoundedApiKeyStoreTests StoreCacheSeconds = cacheSeconds, }), log ?? NullLogger.Instance, - clock ?? TimeProvider.System); + clock ?? TimeProvider.System, + guard); private static ApiKeyOpcUaUserAuthenticator Auth(IApiKeyStore store) => new( store, From 6aa5edaf309cc94aa2786783691674688f9dcd05 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:26:12 -0400 Subject: [PATCH 14/27] fix(scripting): only timeouts are quarantine strikes; alarm Bad annotation once per bucket change (fix round 1 I1, I3) - I1 ScriptRunGuard: a run refused because the previous one is still executing is a Bad publish plus otopcua.script.run_refused, and no longer a quarantine strike. Only a timeout (and an allocation breach) counts; a refusal neither adds to nor resets the timeout streak; a completed run (value or throw) resets it. On a 1 Hz trigger with a 2 s budget a single 4 s overrun is one timeout plus two refusals and must not quarantine. Tests updated across the guard, the alarm engine, the Calculation evaluator and the VT evaluator, plus Refusals_neither_count_toward_nor_reset_the_timeout_streak. - I3 ScriptedAlarmEngine: the quality bucket is tracked only after the predicate completes (or in the inputs-not-ready path), so the tracked bucket is the last one annotated. A refused or quarantined predicate used to re-track the input bucket Good before the attempt and then flip it Bad, emitting QualityChanged on every push. The run-guard tests now count emissions (exactly 1 across N pushes, refused and quarantined cases), with a healthy sibling's dispatch count as the positive control that every push was processed. Both were red before the fix. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ScriptedAlarmEngine.cs | 11 ++- .../ScriptRunGuard.cs | 31 +++++---- .../ScriptingOptions.cs | 7 +- .../CalculationDriverFactoryExtensions.cs | 2 +- .../CalculationEvaluator.cs | 2 +- .../Engines/RoslynVirtualTagEvaluator.cs | 2 +- .../ScriptedAlarmEngineRunGuardTests.cs | 67 +++++++++++++++--- .../ScriptRunGuardTests.cs | 69 +++++++++++++++---- .../CalculationEvaluatorRunGuardTests.cs | 25 +++---- .../RoslynVirtualTagEvaluatorRunGuardTests.cs | 12 ++-- 10 files changed, 161 insertions(+), 67 deletions(-) diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index 9438132a..ae6352f5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -76,7 +76,7 @@ public sealed class ScriptedAlarmEngine : IDisposable /// /// Per-alarm run quarantine (keyed by alarm id, source identity = predicate source). Refuses a /// predicate run while the alarm's previous (timed-out, abandoned) run is still executing and - /// quarantines after three consecutive non-completions or an allocation-budget breach — one orphan + /// quarantines after three consecutive timeouts (a refusal is not a strike) or an allocation-budget breach — one orphan /// per alarm instead of one per upstream push. Reset on every (a redeploy). /// private readonly ScriptRunGuard _guard; @@ -798,7 +798,6 @@ public sealed class ScriptedAlarmEngine : IDisposable // short-circuit so an outright-Bad input is still observed. A bucket change with no state // transition is delivered out of band as a QualityChanged emission (see below). var worstStatus = WorstInputStatus(scratch.ReadCache); - var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus); // Cold-start guard — skip the predicate when any referenced upstream tag has no // cached value yet (the upstream subscription hasn't delivered its first push). @@ -811,7 +810,7 @@ public sealed class ScriptedAlarmEngine : IDisposable // The condition is frozen (can't trust its state), but its source quality just changed // bucket — annotate it out of band so a comms-lost / Bad-input scripted condition reports // Bad, mirroring the native OT path. - if (qualityBucketChanged) + if (TrackQualityBucket(state.Definition.AlarmId, worstStatus)) pendingEmissions.Add(BuildQualityEmission(state, seed, worstStatus)); return seed; } @@ -851,6 +850,12 @@ public sealed class ScriptedAlarmEngine : IDisposable } scratch.LastNonCompletion = null; + // Track the bucket only once the predicate has COMPLETED (I3). The tracked bucket is the last + // bucket this alarm was annotated with; tracking the input bucket before the attempt flipped it to + // Good on every push, so a refused / quarantined predicate (which annotates Bad) re-emitted + // QualityChanged on every push instead of once per real bucket change. + var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus); + var result = Part9StateMachine.ApplyPredicate(seed, predicateTrue, nowUtc); var transition = result.Emission != EmissionKind.None ? BuildEmission(state, result.State, result.Emission, worstStatus) diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs index 8f1acefd..dc308ce6 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs @@ -24,8 +24,10 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// (), counted on /// . One orphan per script, never one per trigger. /// Quarantines after -/// consecutive non-completions (timeouts and still-running refusals both count; a run that -/// returns — a value or a throw — resets the count): no further dispatch, every attempt throws +/// consecutive timeouts (only a timeout is a strike; a refusal is a Bad publish plus its counter +/// and leaves the streak untouched — on a 1 Hz trigger with a 2 s budget one 4 s overrun is a +/// timeout plus two refusals and must not quarantine; a run that returns — a value or a throw — +/// resets the streak): no further dispatch, every attempt throws /// , one Error log, one otopcua.script.quarantined /// increment, one event. Lifted by a source change (the operator fixed the /// script — a new source identity under the same key starts clean and the old orphan no longer blocks @@ -177,10 +179,12 @@ public sealed class ScriptRunGuard // not) still holds the slot is refused here, before anything is dispatched to the pool. if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0) { + // A refusal is a Bad publish plus its counter — NOT a strike. Only a timeout (and an allocation + // breach) counts toward quarantine: on a 1 Hz trigger with a 2 s budget a single 4 s overrun is + // one timeout followed by two refusals, and counting those would quarantine a script that merely + // ran long once. The refusal also leaves the timeout streak untouched. Interlocked.Increment(ref _refused); RefusedCounter.Add(1, new KeyValuePair("script", key)); - if (NoteNonCompletion(key, state, "refused: previous evaluation still running")) - throw new ScriptQuarantinedException(key, state.QuarantineReason!); throw new ScriptRunRefusedException(key); } @@ -241,7 +245,7 @@ public sealed class ScriptRunGuard // The caller's cancel wins even when it lands a few µs after the timeout fired. if (ct.IsCancellationRequested) throw new OperationCanceledException(ct); runCts.Cancel(); - if (NoteNonCompletion(key, state, $"timed out after {timeout.TotalMilliseconds:F1} ms")) + if (NoteTimeout(key, state, timeout)) throw new ScriptQuarantinedException(key, state.QuarantineReason!); throw new ScriptTimeoutException(timeout); } @@ -252,11 +256,11 @@ public sealed class ScriptRunGuard catch (Exception) { // The script returned control (by throwing) — that is a completion for quarantine purposes. - Interlocked.Exchange(ref state.ConsecutiveNonCompletions, 0); + Interlocked.Exchange(ref state.ConsecutiveTimeouts, 0); throw; } - Interlocked.Exchange(ref state.ConsecutiveNonCompletions, 0); + Interlocked.Exchange(ref state.ConsecutiveTimeouts, 0); if (BreachedBudget(slot.AllocatedBytes)) { // The continuation above has already (or will have) quarantined the key; make the reason visible @@ -291,13 +295,14 @@ public sealed class ScriptRunGuard } } - /// Record a non-completion; returns true when it tipped the key into quarantine. - private bool NoteNonCompletion(string key, KeyState state, string what) + /// Record a timeout strike; returns true when it tipped the key into quarantine. A completed run + /// (value or throw) resets the streak; a refusal neither counts nor resets. + private bool NoteTimeout(string key, KeyState state, TimeSpan timeout) { - var count = Interlocked.Increment(ref state.ConsecutiveNonCompletions); + var count = Interlocked.Increment(ref state.ConsecutiveTimeouts); if (count < _quarantineThreshold) return false; Quarantine(key, state, - $"{count} consecutive evaluation(s) did not complete (last: {what}); " + + $"{count} consecutive evaluation(s) timed out (budget {timeout.TotalMilliseconds:F0} ms); " + "the script is not evaluated again until its source changes or it is redeployed", "timeouts"); return true; @@ -333,7 +338,7 @@ public sealed class ScriptRunGuard { public string? SourceIdentity = sourceIdentity; public int InFlight; - public int ConsecutiveNonCompletions; + public int ConsecutiveTimeouts; public string? QuarantineReason; } @@ -368,7 +373,7 @@ public sealed class ScriptRunRefusedException : Exception /// /// Thrown by for a quarantined script: repeated -/// non-completion or an allocation-budget breach. Nothing is dispatched while quarantined; the seam +/// timeouts or an allocation-budget breach. Nothing is dispatched while quarantined; the seam /// publishes Bad carrying the reason, which an operator clears by fixing the script (source change) or /// redeploying. /// diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs index ba1399fa..f101a0e1 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs @@ -33,10 +33,11 @@ public sealed class ScriptingOptions public long MaxAllocBytesPerRun { get; set; } = DefaultMaxAllocBytesPerRun; /// - /// Consecutive non-completions (a run that times out, or a run refused because the previous one is - /// still executing) after which the script is quarantined: no further runs are dispatched, every + /// Consecutive timeouts after which the script is quarantined: no further runs are dispatched, every /// evaluation reports Bad, and the quarantine lifts only on a source change, a redeploy or an explicit - /// reset. Default 3; values below 1 are treated as 1. + /// reset. Only a timeout is a strike — a run refused because the previous one is still executing is + /// reported Bad and counted, but neither adds to nor resets the streak; a completed run (value or + /// throw) resets it. Default 3; values below 1 are treated as 1. /// public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts; } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs index 349a0cf1..1a589d65 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs @@ -107,7 +107,7 @@ public static class CalculationDriverFactoryExtensions /// Optional run-guard allocation budget per script run, in bytes (default 64 MB; ≤ 0 disables). public long? MaxAllocBytesPerRun { get; init; } - /// Optional run-guard quarantine threshold: consecutive non-completions before a tag's script stops being evaluated (default 3). + /// Optional run-guard quarantine threshold: consecutive timeouts (refusals do not count) before a tag's script stops being evaluated (default 3). public int? QuarantineAfterConsecutiveTimeouts { get; init; } } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs index ba4931c7..e47cc745 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs @@ -42,7 +42,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// Run quarantine. Every compiled run goes through one keyed by /// calc-tag RawPath with the script source as the identity: a run is refused while the tag's previous /// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive -/// non-completions or an allocation-budget breach — both surface as a +/// timeouts (a refusal is Bad but not a strike) or an allocation-budget breach — both surface as a /// the driver maps to Bad. The driver's evaluation pump awaits rather than /// blocking a pool thread on the sync , /// and the guard starts the budget clock only once the pool has picked the script up, so pool starvation diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs index c6823c41..f0bbd150 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs @@ -35,7 +35,7 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines; /// Run quarantine. Every compiled run goes through one keyed by /// virtual-tag id with the expression as the source identity: a run is refused while the tag's previous /// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive -/// non-completions or an allocation-budget breach. Both surface as +/// timeouts (a refusal is Bad but not a strike) or an allocation-budget breach. Both surface as /// so VirtualTagActor publishes Bad exactly as it does for a timeout. A quarantine lifts when the /// expression changes, on / (the deploy /// boundary), never by itself. diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs index 4074d0d9..ab2fde04 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs @@ -9,8 +9,8 @@ namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; /// /// runs every predicate through a keyed by /// alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation -/// spawned another abandoned run); now the orphan is one, later pushes are refused and the alarm is -/// quarantined — frozen at its prior state and annotated Bad quality so the operator can see it — while a +/// spawned another abandoned run); now the orphan is one, later pushes are refused (and a +/// predicate that times out repeatedly is quarantined) — frozen at its prior state and annotated Bad quality so the operator can see it — while a /// sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine. /// [Trait("Category", "Unit")] @@ -36,7 +36,7 @@ public sealed class ScriptedAlarmEngineRunGuardTests } [Fact] - public async Task Runaway_predicate_leaks_one_orphan_is_quarantined_annotated_Bad_and_a_sibling_keeps_evaluating() + public async Task Runaway_predicate_leaks_one_orphan_is_refused_annotated_Bad_once_and_a_sibling_keeps_evaluating() { var up = new FakeUpstream(); up.Set("Temp", 50); @@ -50,20 +50,27 @@ public sealed class ScriptedAlarmEngineRunGuardTests Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")], TestContext.Current.CancellationToken); - // Two more pushes: refused (orphan still running), then quarantined. - up.Push("Temp", 60); - await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= 1); - up.Push("Temp", 70); - await WaitForAsync(() => eng.RunGuard.IsQuarantined("runaway", out _)); + // N more pushes while the orphan runs: every one is refused (a refusal is not a strike — I1), and + // the Bad annotation is emitted ONCE for the whole streak, not once per push (I3). + const int pushes = 4; + for (var i = 0; i < pushes; i++) + { + up.Push("Temp", 60 + i); + var expected = i + 1; + await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= expected); + } + await WaitForAsync(() => eng.RunGuard.StartedRunCount >= 1 + 1 + pushes); - eng.RunGuard.StartedRunCount.ShouldBe(1 + 3, "one dispatch for the runaway alarm, one per evaluation for the healthy one"); + eng.RunGuard.IsQuarantined("runaway", out _).ShouldBeFalse("refusals are not strikes"); + eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one dispatch for the runaway alarm, one per evaluation for the healthy one"); eng.RunGuard.InFlightCount.ShouldBe(1); - // The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving. + // The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving — + // exactly once across the whole run of non-completions. ScriptedAlarmEvent[] snapshot; lock (events) snapshot = events.ToArray(); var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray(); - quality.ShouldNotBeEmpty(); + quality.Length.ShouldBe(1); quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError); eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented"); @@ -72,6 +79,44 @@ public sealed class ScriptedAlarmEngineRunGuardTests await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active); } + /// I3 — a QUARANTINED predicate is re-attempted on every push and refused before dispatch; the + /// Bad annotation must still be emitted once, not once per push. + [Fact] + public async Task A_quarantined_predicate_emits_QualityChanged_once_across_many_pushes() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + var events = new List(); + eng.OnEvent += (_, e) => { lock (events) events.Add(e); }; + + await eng.LoadAsync( + [Alarm("q", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"), + Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 1000;""")], + TestContext.Current.CancellationToken); + eng.RunGuard.IsQuarantined("q", out _).ShouldBeTrue(); + + // Each push re-evaluates BOTH alarms (same input). The healthy sibling's dispatch count is the + // positive control proving every push was actually processed — so the quarantined alarm was + // re-attempted N times, not coalesced into one. + const int pushes = 5; + for (var i = 0; i < pushes; i++) + { + var before = eng.RunGuard.StartedRunCount; + up.Push("Temp", 60 + i); + await WaitForAsync(() => eng.RunGuard.StartedRunCount > before); + } + eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one runaway dispatch, then healthy-only dispatches"); + await Task.Delay(300); // absence window for trailing emissions + + ScriptedAlarmEvent[] snapshot; + lock (events) snapshot = events.ToArray(); + var quality = snapshot.Where(e => e.AlarmId == "q" && e.Emission == EmissionKind.QualityChanged).ToArray(); + quality.Length.ShouldBe(1, "one Bad annotation for the quarantine, not one per push"); + quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError); + snapshot.ShouldNotContain(e => e.AlarmId == "healthy" && e.Emission == EmissionKind.QualityChanged); + } + [Fact] public async Task A_reload_lifts_the_quarantine() { diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs index 974e6a7c..f60045b9 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs @@ -9,7 +9,7 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; /// Calculation evaluator and the scripted-alarm engine. A timed-out script is abandoned, not stopped, so /// before the guard every re-trigger of a runaway script leaked one more thread-pool thread; the guard /// refuses to start a run while the previous one is still executing, quarantines a script after three -/// consecutive non-completions, and quarantines a script whose single run allocates past the budget. +/// consecutive timeouts (a refusal is not a strike), and quarantines a script whose single run allocates past the budget. /// [Trait("Category", "Unit")] public sealed class ScriptRunGuardTests @@ -28,6 +28,14 @@ public sealed class ScriptRunGuardTests private const string FastScript = "return 42;"; + /// Outlives the 60 ms budget but finishes within ~250 ms, so the slot frees up between attempts. + private const string BriefSpinScript = + """ + var end = System.DateTime.UtcNow.AddMilliseconds(250); + while (System.DateTime.UtcNow < end) { } + return 1; + """; + private static readonly TimeSpan ShortTimeout = TimeSpan.FromMilliseconds(60); private static ScriptEvaluator Compile(string source) => @@ -83,17 +91,21 @@ public sealed class ScriptRunGuardTests } [Fact] - public async Task Three_consecutive_non_completions_quarantine_the_script_and_a_sibling_keeps_evaluating() + public async Task Three_consecutive_timeouts_quarantine_the_script_and_a_sibling_keeps_evaluating() { var guard = new ScriptRunGuard(); - using var spin = Compile(SpinScript); + using var spin = Compile(BriefSpinScript); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; - // 1: times out, 2: refused (orphan still running), 3: refused → threshold reached → quarantined. - await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); - await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); - var quarantined = await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); + // Three timeouts in a row — each orphan is allowed to finish so the next attempt is a real run, + // not a refusal (a refusal is not a strike; see the next test). + await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + await WaitForIdle(guard); + await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + await WaitForIdle(guard); + var quarantined = await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + await WaitForIdle(guard); quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); quarantined.Message.ShouldContain("3 consecutive"); @@ -102,19 +114,46 @@ public sealed class ScriptRunGuardTests guard.QuarantinedCount.ShouldBe(1); // A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run. - await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); - guard.StartedRunCount.ShouldBe(1); + await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + guard.StartedRunCount.ShouldBe(3); // Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key. (await Run(guard, "good", FastScript, fast, ct: ct)).ShouldBe(42); guard.IsQuarantined("good", out _).ShouldBeFalse(); - guard.StartedRunCount.ShouldBe(2); + guard.StartedRunCount.ShouldBe(4); + } + + /// + /// A refusal is NOT a strike (review fix round 1, I1): on a 1 Hz trigger with a 2 s budget a single + /// 4 s overrun is one timeout followed by two refusals, and counting those would quarantine a script + /// that merely ran long once. A refusal is a Bad publish plus its counter, and it leaves the timeout + /// streak untouched — neither adding to it nor resetting it. + /// + [Fact] + public async Task Refusals_neither_count_toward_nor_reset_the_timeout_streak() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 }); + using var spin = Compile(BriefSpinScript); + var ct = TestContext.Current.CancellationToken; + + // timeout #1, then refusals while the orphan is still running — however many, no quarantine. + await Should.ThrowAsync(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct)); + for (var i = 0; i < 5; i++) + await Should.ThrowAsync(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeFalse("refusals are not strikes"); + guard.RefusedWhileRunningCount.ShouldBe(5); + await WaitForIdle(guard); + + // …and they did not RESET the streak either: the second real timeout reaches the threshold of 2. + await Should.ThrowAsync(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeTrue(); + await WaitForIdle(guard); } /// /// The thread-leak regression. Pre-guard, a while(true) script on a 1 Hz trigger leaked one /// pool thread per trigger — 30 orphans after 30 triggers, trending toward pool starvation. Now the - /// first trigger's orphan is the only one, every later trigger is refused or quarantined, and a healthy + /// first trigger's orphan is the only one, every later trigger is refused, and a healthy /// sibling keeps evaluating throughout (the positive control an absence assertion needs). /// /// @@ -145,7 +184,9 @@ public sealed class ScriptRunGuardTests guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing"); guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling"); - guard.IsQuarantined("runaway", out _).ShouldBeTrue(); + // One timeout + 29 refusals: not quarantined (a refusal is not a strike), but never re-dispatched. + guard.IsQuarantined("runaway", out _).ShouldBeFalse(); + guard.RefusedWhileRunningCount.ShouldBe(29); healthyResults.ShouldAllBe(v => v == 42); healthyResults.Count.ShouldBe(30); @@ -156,13 +197,11 @@ public sealed class ScriptRunGuardTests [Fact] public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot() { - var guard = new ScriptRunGuard(); + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); using var spin = Compile(SpinScript); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; - await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); - await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); guard.IsQuarantined("vt", out _).ShouldBeTrue(); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs index 85943f16..679e7f71 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs @@ -10,8 +10,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests; /// runs every compiled script through a /// keyed by calc-tag RawPath. A timed-out run is abandoned on its pool thread; before the guard, every /// later trigger of the same tag stacked another orphan behind it (one leaked thread per trigger). Now -/// the next trigger is refused ("previous evaluation still running"), the third consecutive -/// non-completion quarantines the tag, and a healthy sibling on the same evaluator keeps computing. +/// every later trigger is refused ("previous evaluation still running" — Bad, but not a quarantine +/// strike; only consecutive timeouts quarantine), and a healthy sibling on the same evaluator keeps computing. /// public sealed class CalculationEvaluatorRunGuardTests { @@ -24,7 +24,7 @@ public sealed class CalculationEvaluatorRunGuardTests private static readonly Dictionary NoDeps = new(); [Fact] - public async Task A_trigger_during_the_previous_orphan_is_refused_then_the_tag_is_quarantined_and_a_sibling_keeps_computing() + public async Task Triggers_during_the_previous_orphan_are_refused_not_quarantined_and_a_sibling_keeps_computing() { using var sut = new CalculationEvaluator( NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60)); @@ -35,15 +35,16 @@ public sealed class CalculationEvaluatorRunGuardTests first.Reason.ShouldNotBeNull(); first.Reason.ShouldContain("timed out"); - var second = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); - second.Success.ShouldBeFalse(); - second.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); - - var third = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); - third.Success.ShouldBeFalse(); - third.Reason.ShouldNotBeNull(); - third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); - sut.QuarantinedScripts.ShouldBe(new[] { "calc/loop" }); + // Every later trigger while the orphan runs is a Bad "previous evaluation still running" — a + // refusal is not a strike, so the tag is NOT quarantined by them (fix round 1, I1). + for (var i = 0; i < 3; i++) + { + var refused = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); + refused.Success.ShouldBeFalse(); + refused.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + } + sut.QuarantinedScripts.ShouldBeEmpty(); + sut.RunGuard.RefusedWhileRunningCount.ShouldBe(3); // Exactly one run was ever dispatched for the runaway tag — the leak is one orphan, not one per trigger. sut.RunGuard.StartedRunCount.ShouldBe(1); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs index f0de4999..cd18ae5e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs @@ -10,7 +10,7 @@ namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; /// /// runs every compiled expression through a /// keyed by virtual-tag id: a runaway expression leaks one orphaned pool -/// thread, not one per trigger, the tag is quarantined after three consecutive non-completions (surfacing +/// thread, not one per trigger, later triggers are refused, the tag is quarantined after three consecutive timeouts (surfacing /// as a Failure the actor publishes Bad for), and the quarantine lifts at the deploy boundary. /// public sealed class RoslynVirtualTagEvaluatorRunGuardTests @@ -23,7 +23,7 @@ public sealed class RoslynVirtualTagEvaluatorRunGuardTests private static readonly Dictionary NoDeps = new(); [Fact] - public void Runaway_expression_is_refused_then_quarantined_and_a_sibling_keeps_evaluating() + public void Runaway_expression_is_refused_while_its_orphan_runs_and_a_sibling_keeps_evaluating() { using var sut = new RoslynVirtualTagEvaluator( NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60)); @@ -33,12 +33,10 @@ public sealed class RoslynVirtualTagEvaluatorRunGuardTests first.Reason.ShouldNotBeNull(); first.Reason.ShouldContain("timed out"); + // Refusals, not strikes (fix round 1, I1): the tag stays Bad but is not quarantined by them. sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); - - var third = sut.Evaluate("vt-loop", SpinScript, NoDeps); - third.Reason.ShouldNotBeNull(); - third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); - sut.QuarantinedScripts.ShouldBe(new[] { "vt-loop" }); + sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + sut.QuarantinedScripts.ShouldBeEmpty(); sut.RunGuard.StartedRunCount.ShouldBe(1, "one orphan, not one per trigger"); var sibling = sut.Evaluate("vt-ok", "return 42;", NoDeps); From ad6be56850f2c1551737014d0f6c6c224b864d94 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:31:41 -0400 Subject: [PATCH 15/27] fix(runtime): bound DependencyMux last-value retention to replay-opting refs (fix round 1 I2) DependencyMuxActor retained the last value of every ref ever registered, and never removed the entries. The continuous-historization recorder registers every historized ref, so on a historizing node the map grew with the historized address space and never shrank. The class remark claimed a bound that did not hold. - Retain only refs that at least one subscriber registered with ReplayRetained: true names (VT actors, the Calculation mux adapter, the scripted-alarm host). A non-opting subscriber is still delivered to but retains nothing. - When a ref's last replay-opting subscriber leaves, its value is kept for a grace (default 2 min, Props(retentionGrace)) so a restart (old adapter gone, new one not yet registered) still replays. A periodic sweep then drops it along with the entry. Entries with no subscriber and nothing retained are removed. - New SourceRemoved(DriverInstanceId) evicts the values a removed driver produced, so its last Good is never replayed. The mux records the publishing driver per retained ref. No production sender exists yet: DriverHostActor is outside this lane (follow-up). - Remark rewritten. Tests: retention bounded to replay-opting refs, release after grace (with an in-grace replay as the positive control), SourceRemoved eviction, and the restart-replay scenario still green. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../VirtualTags/DependencyMuxActor.cs | 251 ++++++++++++++---- .../VirtualTags/DependencyMuxReplayTests.cs | 95 ++++++- 2 files changed, 286 insertions(+), 60 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs index 3af36d6a..40e9bf19 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs @@ -28,53 +28,132 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; /// would over-emit if it spanned the cluster. /// /// -/// Last-value retention + replay (R07:H6). For every ref that has EVER been registered the -/// mux retains the last (value, status, timestamp) it forwarded — through unregisters too, -/// because the case that matters is a subscriber that went away and came back: a Calculation -/// driver Restart is a new driver instance + a new mux adapter, and before this every calc tag sat -/// BadWaitingForInitialData until each of its inputs happened to change again (a -/// scripted-alarm host restart has the same shape). A with -/// receives one batch of the retained values for its -/// refs, in the same shape a driver burst would arrive in. Opt-in per registration: the -/// continuous-historization recorder re-registers on every deploy and a replay there would -/// record a stale sample per tag per deploy. Memory is bounded by the number of refs scripts have -/// ever named on this node, not by the address space. +/// Last-value retention + replay (R07:H6, bounded in review fix round 1 / I2). The mux +/// retains the last (value, status, timestamp) of a ref only while at least one subscriber +/// registered with names it (the virtual-tag +/// actors, the Calculation mux adapter, the scripted-alarm host). A registration that asks receives +/// one batch of the retained values for its refs, in the shape a driver burst arrives in. The case +/// this serves is a subscriber that goes away and comes back: a Calculation driver Restart is a new +/// driver instance + a new mux adapter, and before this every calc tag sat +/// BadWaitingForInitialData until each of its inputs happened to change again. So when a +/// ref's last replay-opting subscriber leaves, its value is kept for +/// (default ; a returning replay registration inside it cancels +/// the release), then dropped together with the ref's entry. +/// +/// +/// A subscriber that does NOT opt in retains nothing: the continuous-historization recorder +/// registers every historized ref on the node (a replay there would record a stale sample per tag +/// per deploy), and retaining on its behalf made the map grow with the historized address space and +/// never shrink. Memory is therefore bounded by the refs replay-opting subscribers currently name, +/// plus those released within the last grace window. A ref with no subscriber and nothing retained +/// has no entry at all. +/// +/// +/// Source removal. The mux records which driver instance last published each retained ref; +/// evicts every value that driver produced, so a removed driver's last +/// Good is never replayed as current. A comms-loss fan (a Bad batch) already overwrites the retained +/// value with Bad. ⚠️ As of fix round 1 no production actor sends yet — +/// DriverHostActor (outside the scripting lane) must Tell it when it stops / removes a driver +/// child. Until then a removed driver's last value can still be replayed to a replay-opting +/// subscriber that re-registers for that ref while another replay-opting subscriber holds it, or +/// within the grace window. /// /// -public sealed class DependencyMuxActor : ReceiveActor +public sealed class DependencyMuxActor : ReceiveActor, IWithTimers { public const string ActorName = "dependency-mux"; + /// How long a ref's retained value outlives its last replay-opting subscriber. + public static readonly TimeSpan DefaultRetentionGrace = TimeSpan.FromMinutes(2); + /// Register a subscriber's interest in a set of tag refs. Idempotent on re-register — /// the second call replaces the prior interest set for that subscriber. /// The refs the subscriber wants. /// The subscriber. - /// When true, the mux immediately sends the subscriber one - /// holding the last retained value of every ref - /// in it has one for (nothing when it has none). + /// When true, the mux retains the last value of every ref in + /// while this subscriber is registered, and immediately sends the subscriber one + /// holding the last retained value of every ref it has + /// one for (nothing when it has none). public sealed record RegisterInterest(IReadOnlyList TagRefs, IActorRef Subscriber, bool ReplayRetained = false); /// Unregister every interest held by . Sent on PostStop by /// the subscriber, or by Terminated handling when the mux watches. public sealed record UnregisterInterest(IActorRef Subscriber); - // A ref stays in _byRef once registered — with an EMPTY set after its last subscriber leaves — so - // the hot path's single lookup per item also decides retention. _byRef.Count == 0 keeps meaning - // "nothing was ever registered", which the batch path early-outs on. - private readonly Dictionary> _byRef = new(StringComparer.Ordinal); - private readonly Dictionary> _bySubscriber = new(); - private readonly Dictionary _last = new(StringComparer.Ordinal); + /// A driver instance was removed / stopped: evict every retained value it produced so it is + /// never replayed. Subscriptions are untouched. + /// The removed driver instance. + public sealed record SourceRemoved(string DriverInstanceId); + + private sealed class SweepReleased + { + public static readonly SweepReleased Instance = new(); + } + + private const string SweepTimerKey = "sweep-released"; + + /// Per-ref state. One lookup on the hot path answers both "who wants it" and "retain it?". + private sealed class RefEntry + { + public readonly HashSet Subscribers = new(); + + /// Registered subscribers of this ref that opted into replay. + public int Replayers; + + /// When the last replay-opting subscriber left (retention in grace), or null. + public DateTime? ReleasedAtUtc; + + public VirtualTagActor.DependencyValueChanged? Last; + public string? LastSource; + + public bool Retains => Replayers > 0 || ReleasedAtUtc is not null; + public bool IsDead => Subscribers.Count == 0 && Replayers == 0 && Last is null; + + public void Forget() + { + Last = null; + LastSource = null; + ReleasedAtUtc = null; + } + } + + private sealed record Registration(HashSet Refs, bool Replay); + + private readonly Dictionary _byRef = new(StringComparer.Ordinal); + private readonly Dictionary _bySubscriber = new(); private readonly ILoggingAdapter _log = Context.GetLogger(); + /// How long a released ref's value is kept. + public TimeSpan RetentionGrace { get; } + + /// + public ITimerScheduler Timers { get; set; } = null!; + + /// Test hook — refs that currently hold a retained value. + internal int RetainedRefCount => _byRef.Values.Count(e => e.Last is not null); + + /// Test hook — refs the mux holds any state for (subscribers or a retained value). + internal int TrackedRefCount => _byRef.Count; + /// Creates props for the DependencyMuxActor. + /// How long a ref's value outlives its last replay-opting subscriber; + /// null ⇒ . /// The props for creating this actor. - public static Props Props() => Akka.Actor.Props.Create(); + public static Props Props(TimeSpan? retentionGrace = null) + { + var grace = retentionGrace ?? DefaultRetentionGrace; + return Akka.Actor.Props.Create(() => new DependencyMuxActor(grace)); + } /// Initializes a new instance of the class. - public DependencyMuxActor() + /// See ; non-positive ⇒ . + public DependencyMuxActor(TimeSpan retentionGrace) { + RetentionGrace = retentionGrace > TimeSpan.Zero ? retentionGrace : DefaultRetentionGrace; Receive(OnRegister); Receive(msg => RemoveSubscriber(msg.Subscriber)); + Receive(OnSourceRemoved); + Receive(_ => OnSweep()); Receive(OnAttributeValuePublished); Receive(OnAttributeValueBatchPublished); Receive(msg => RemoveSubscriber(msg.ActorRef)); @@ -82,31 +161,30 @@ public sealed class DependencyMuxActor : ReceiveActor private void OnRegister(RegisterInterest msg) { - // Replace any prior interest set so re-registers on actor restart don't leak old refs. - if (_bySubscriber.TryGetValue(msg.Subscriber, out var priorRefs)) - { - foreach (var r in priorRefs) - { - if (_byRef.TryGetValue(r, out var set)) set.Remove(msg.Subscriber); - } - } + // Replace any prior interest set so re-registers on actor restart don't leak old refs. Detaching first + // may put a ref into grace; the re-attach below takes it straight back out. + Detach(msg.Subscriber); var refs = new HashSet(msg.TagRefs, StringComparer.Ordinal); - _bySubscriber[msg.Subscriber] = refs; + _bySubscriber[msg.Subscriber] = new Registration(refs, msg.ReplayRetained); List? replay = null; foreach (var r in refs) { - if (!_byRef.TryGetValue(r, out var set)) + if (!_byRef.TryGetValue(r, out var entry)) { - set = new HashSet(); - _byRef[r] = set; + entry = new RefEntry(); + _byRef[r] = entry; } - set.Add(msg.Subscriber); - if (msg.ReplayRetained && _last.TryGetValue(r, out var retained)) + entry.Subscribers.Add(msg.Subscriber); + if (!msg.ReplayRetained) continue; + entry.Replayers++; + entry.ReleasedAtUtc = null; + if (entry.Last is { } retained) (replay ??= []).Add(retained); } Context.Watch(msg.Subscriber); - _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs", msg.Subscriber, refs.Count); + _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs (replay {Replay})", + msg.Subscriber, refs.Count, msg.ReplayRetained); // One batch, after the registration is complete, in exactly the shape a driver burst arrives in — // so the subscriber's hot path handles it with no special case. @@ -119,29 +197,89 @@ public sealed class DependencyMuxActor : ReceiveActor private void RemoveSubscriber(IActorRef subscriber) { - if (!_bySubscriber.TryGetValue(subscriber, out var refs)) return; - foreach (var r in refs) - { - // The ref's entry stays (possibly empty) so its last value keeps being retained for a - // subscriber that comes back — see the class remarks. - if (_byRef.TryGetValue(r, out var set)) set.Remove(subscriber); - } - _bySubscriber.Remove(subscriber); + if (!Detach(subscriber)) return; Context.Unwatch(subscriber); _log.Debug("DependencyMux: subscriber {Sub} removed", subscriber); } + /// Drop 's registration. A ref losing its last replay-opting + /// subscriber goes into grace; a ref left with no state at all is removed. + private bool Detach(IActorRef subscriber) + { + if (!_bySubscriber.Remove(subscriber, out var registration)) return false; + var now = DateTime.UtcNow; + var released = false; + foreach (var r in registration.Refs) + { + if (!_byRef.TryGetValue(r, out var entry)) continue; + entry.Subscribers.Remove(subscriber); + if (registration.Replay && --entry.Replayers == 0 && entry.Last is not null) + { + entry.ReleasedAtUtc = now; + released = true; + } + if (entry.IsDead) _byRef.Remove(r); + } + if (released && !Timers.IsTimerActive(SweepTimerKey)) + Timers.StartPeriodicTimer(SweepTimerKey, SweepReleased.Instance, RetentionGrace); + return true; + } + + /// Drop retained values whose grace has run out — with a periodic sweep at the grace interval a + /// released value lives between one and two grace periods — and stop the sweep once nothing is in grace. + private void OnSweep() + { + var cutoff = DateTime.UtcNow - RetentionGrace; + var stillInGrace = false; + List? dead = null; + foreach (var (r, entry) in _byRef) + { + if (entry.ReleasedAtUtc is not { } releasedAt) continue; + if (releasedAt > cutoff) + { + stillInGrace = true; + continue; + } + entry.Forget(); + if (entry.IsDead) (dead ??= []).Add(r); + } + if (dead is not null) + foreach (var r in dead) _byRef.Remove(r); + if (!stillInGrace) Timers.Cancel(SweepTimerKey); + } + + private void OnSourceRemoved(SourceRemoved msg) + { + List? dead = null; + var evicted = 0; + foreach (var (r, entry) in _byRef) + { + if (entry.Last is null || !string.Equals(entry.LastSource, msg.DriverInstanceId, StringComparison.Ordinal)) + continue; + entry.Forget(); + evicted++; + if (entry.IsDead) (dead ??= []).Add(r); + } + if (dead is not null) + foreach (var r in dead) _byRef.Remove(r); + _log.Debug("DependencyMux: source {Driver} removed — evicted {Count} retained value(s)", msg.DriverInstanceId, evicted); + } + private void OnAttributeValuePublished(DriverInstanceActor.AttributeValuePublished msg) { - if (!_byRef.TryGetValue(msg.FullReference, out var subscribers)) + if (!_byRef.TryGetValue(msg.FullReference, out var entry)) { - // No virtual tag ever cared about this ref — drop. Common in normal operation; the address + // Nothing is registered for (or retaining) this ref — drop. Common in normal operation; the address // space carries thousands of tags and only a fraction feed virtual-tag expressions. return; } var dep = new VirtualTagActor.DependencyValueChanged(msg.FullReference, msg.Value, msg.TimestampUtc, msg.StatusCode); - _last[msg.FullReference] = dep; - foreach (var sub in subscribers) + if (entry.Retains) + { + entry.Last = dep; + entry.LastSource = msg.DriverInstanceId; + } + foreach (var sub in entry.Subscribers) { // A single-item batch, so the mux emits exactly ONE message shape and every subscriber needs // only the batch receive on its hot path. One list per subscriber: batch ownership transfers @@ -153,7 +291,7 @@ public sealed class DependencyMuxActor : ReceiveActor private void OnAttributeValueBatchPublished(DriverInstanceActor.AttributeValueBatchPublished msg) { - // Nobody has registered anything — skip the whole burst rather than probing an empty map per item. + // Nothing registered and nothing retained — skip the whole burst rather than probing an empty map per item. if (_byRef.Count == 0) return; // Allocated lazily on the first matched item: a burst in which nothing is registered for — still @@ -166,10 +304,15 @@ public sealed class DependencyMuxActor : ReceiveActor for (var i = 0; i < count; i++) { var item = msg.Items[i]; - if (!_byRef.TryGetValue(item.FullReference, out var subscribers)) continue; + if (!_byRef.TryGetValue(item.FullReference, out var entry)) continue; var dep = new VirtualTagActor.DependencyValueChanged(item.FullReference, item.Value, item.TimestampUtc, item.StatusCode); - _last[item.FullReference] = dep; - if (subscribers.Count == 0) continue; // retained for a returning subscriber; nobody to forward to now + if (entry.Retains) + { + entry.Last = dep; + entry.LastSource = msg.DriverInstanceId; + } + var subscribers = entry.Subscribers; + if (subscribers.Count == 0) continue; // retained (in grace) for a returning subscriber; nobody to forward to now perSubscriber ??= new Dictionary>(); foreach (var sub in subscribers) { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs index ae42edb9..f761edd9 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs @@ -12,10 +12,13 @@ using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags; /// -/// R07:H6 — the mux retains the last value per ever-registered ref and replays it to a registration -/// that asks (). A Calculation driver -/// Restart (new driver + new adapter) therefore republishes from current inputs instead of sitting -/// BadWaitingForInitialData until every input happens to change. +/// R07:H6 — the mux retains the last value of every ref a REPLAY-OPTING subscriber +/// () is registered for, keeps it for a +/// short grace after the last such subscriber leaves, and replays it to a registration that asks. A +/// Calculation driver Restart (new driver + new adapter) therefore republishes from current inputs instead +/// of sitting BadWaitingForInitialData until every input happens to change. Fix round 1 (I2): a +/// non-opting subscriber (the historization recorder) retains nothing, retention is released after the +/// grace, and a removed source's values are evicted so its last Good is never replayed. /// public sealed class DependencyMuxReplayTests : RuntimeActorTestBase { @@ -27,7 +30,7 @@ public sealed class DependencyMuxReplayTests : RuntimeActorTestBase { var mux = Sys.ActorOf(DependencyMuxActor.Props()); var first = CreateTestProbe(); - mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, first.Ref)); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, first.Ref, ReplayRetained: true)); mux.Tell(Burst(("a", 1), ("b", 2), ("c", 3))); first.ExpectMsg(PresenceBudget).Items.Count.ShouldBe(2); @@ -38,7 +41,7 @@ public sealed class DependencyMuxReplayTests : RuntimeActorTestBase mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "c" }, second.Ref, ReplayRetained: true)); var replay = second.ExpectMsg(PresenceBudget); - replay.Items.Select(i => i.TagId).ShouldBe(new[] { "a" }, "'c' was never registered by anyone, so it is not retained; 'b' is not in this subscriber's refs"); + replay.Items.Select(i => i.TagId).ShouldBe(new[] { "a" }, "'c' was never registered by a replay-opting subscriber, so it is not retained; 'b' is not in this subscriber's refs"); replay.Items[0].Value.ShouldBe(1); } @@ -60,6 +63,86 @@ public sealed class DependencyMuxReplayTests : RuntimeActorTestBase fresh.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); } + /// I2 — a subscriber that did NOT opt into replay (the continuous-historization recorder registers + /// every historized ref) retains nothing; retention is bounded to the refs a replay-opting subscriber named. + [Fact] + public void Only_refs_named_by_a_replay_opting_subscriber_are_retained() + { + var mux = ActorOfAsTestActorRef(DependencyMuxActor.Props()); + var recorder = CreateTestProbe(); + var calc = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "h1", "h2", "h3", "shared" }, recorder.Ref)); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "r", "shared" }, calc.Ref, ReplayRetained: true)); + mux.Tell(Burst(("h1", 1), ("h2", 2), ("h3", 3), ("r", 4), ("shared", 5))); + recorder.ExpectMsg(PresenceBudget).Items.Count.ShouldBe(4, "the recorder is still DELIVERED to"); + calc.ExpectMsg(PresenceBudget); + + mux.UnderlyingActor.RetainedRefCount.ShouldBe(2, "only 'r' and 'shared' — the recorder-only refs retain nothing"); + + var late = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "h1", "r" }, late.Ref, ReplayRetained: true)); + late.ExpectMsg(PresenceBudget) + .Items.Select(i => i.TagId).ShouldBe(new[] { "r" }, "positive control: the opted-in ref replays; 'h1' was never retained"); + + // Unregistering everyone ends up with no entries at all once the grace is over — tested below; here + // the recorder leaving changes nothing about retention. + mux.Tell(new DependencyMuxActor.UnregisterInterest(recorder.Ref)); + mux.UnderlyingActor.RetainedRefCount.ShouldBe(2); + } + + /// I2 — when the last replay-opting subscriber of a ref leaves, its retained value survives a short + /// grace (the restart window: old adapter gone, new one not yet registered) and is then dropped. + [Fact] + public void Retention_is_released_after_the_grace_once_no_replay_subscriber_remains() + { + var mux = ActorOfAsTestActorRef(DependencyMuxActor.Props(TimeSpan.FromMilliseconds(200))); + var gen1 = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen1.Ref, ReplayRetained: true)); + mux.Tell(Burst(("a", 1))); + gen1.ExpectMsg(PresenceBudget); + mux.Tell(new DependencyMuxActor.UnregisterInterest(gen1.Ref)); + + // Positive control: inside the grace a returning subscriber still gets the value. + var gen2 = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen2.Ref, ReplayRetained: true)); + gen2.ExpectMsg(PresenceBudget).Items.Single().Value.ShouldBe(1); + mux.Tell(new DependencyMuxActor.UnregisterInterest(gen2.Ref)); + + // Past the grace: nothing retained, no entry left behind, and a returning subscriber replays nothing. + AwaitAssert(() => + { + mux.UnderlyingActor.RetainedRefCount.ShouldBe(0); + mux.UnderlyingActor.TrackedRefCount.ShouldBe(0); + }, PresenceBudget); + var gen3 = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen3.Ref, ReplayRetained: true)); + gen3.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + } + + /// I2 — a removed source's retained values are evicted, so its last Good is never replayed to a + /// subscriber that (re)registers after the removal. Values from other sources are untouched. + [Fact] + public void SourceRemoved_evicts_that_sources_retained_values_only() + { + var mux = ActorOfAsTestActorRef(DependencyMuxActor.Props()); + var sub = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, sub.Ref, ReplayRetained: true)); + mux.Tell(new DriverInstanceActor.AttributeValueBatchPublished("driver-1", + [new DriverInstanceActor.AttributeValueItem("a", 1, OpcUaStatus.Good, DateTime.UtcNow)])); + mux.Tell(new DriverInstanceActor.AttributeValueBatchPublished("driver-2", + [new DriverInstanceActor.AttributeValueItem("b", 2, OpcUaStatus.Good, DateTime.UtcNow)])); + sub.ExpectMsg(PresenceBudget); + sub.ExpectMsg(PresenceBudget); + + mux.Tell(new DependencyMuxActor.SourceRemoved("driver-1")); + mux.UnderlyingActor.RetainedRefCount.ShouldBe(1); + + var restarted = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, restarted.Ref, ReplayRetained: true)); + restarted.ExpectMsg(PresenceBudget) + .Items.Select(i => i.TagId).ShouldBe(new[] { "b" }, "driver-1 was removed: its last Good must not be replayed"); + } + /// The user-facing scenario: a calc tag computes, the driver is restarted (new instance + new /// adapter) with NO input changing afterwards, and the new instance republishes the current value. [Fact] From 3ddb6876dbe65660c7b2f5e7a5fb3eba4a870151 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:35:54 -0400 Subject: [PATCH 16/27] fix(security): first-party JwtBearer with HS256 pinned; /auth/token mints for a cookie session only The hand-rolled bearer handler is replaced by AddJwtBearer (Microsoft.AspNetCore.Authentication.JwtBearer 10.0.7, pinned on the same servicing line as the other AspNetCore packages). Validation parameters come from JwtTokenService.BuildValidationParameters, which now pins ValidAlgorithms = HS256 for both the bearer scheme and TryValidate; MapInboundClaims is off so the minted Role/zb:username keys resolve. /auth/token requires the cookie scheme, so a bearer can no longer mint a fresh token for itself. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- Directory.Packages.props | 4 + docs/security.md | 13 ++- .../Auth/JwtBearerAuthenticationHandler.cs | 99 ------------------- .../Auth/OtOpcUaAuthSchemes.cs | 39 ++++++++ .../Endpoints/AuthEndpoints.cs | 16 ++- .../Jwt/JwtTokenService.cs | 28 ++---- .../ServiceCollectionExtensions.cs | 18 +++- .../ZB.MOM.WW.OtOpcUa.Security.csproj | 1 + .../AuthEndpointsIntegrationTests.cs | 93 +++++++++++++++++ .../JwtConfigurationGateTests.cs | 2 +- 10 files changed, 187 insertions(+), 126 deletions(-) delete mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 5b95a7e1..ed2dca67 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -35,6 +35,10 @@ + + diff --git a/docs/security.md b/docs/security.md index 6e44569a..fa8269a2 100644 --- a/docs/security.md +++ b/docs/security.md @@ -6,11 +6,14 @@ > auth + role-grant pages live in `OtOpcUa.AdminUI`), and Admin auth policies > register from `OtOpcUa.Host/Program.cs` via `AddOtOpcUaAuth` > (`src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs`) rather -> than in a separate Admin process. The Admin UI uses a **single Cookie -> authentication scheme** — there is no `AddJwtBearer` pipeline. The -> `Security:Jwt` section configures `JwtTokenService`, which mints a JWT at the -> `/auth/token` endpoint for **external** consumers (OPC UA clients / automation -> scripts); the cookie itself stores the `ClaimsPrincipal` directly. DataProtection +> than in a separate Admin process. The Admin UI session is a **Cookie** +> scheme; the cookie stores the `ClaimsPrincipal` directly. JWT is **opt-in by +> `Security:Jwt:SigningKey`** (validated at host start, ≥32 bytes): with a key, +> `/auth/token` mints an HS256 token for a **cookie session only** (a bearer +> cannot renew itself), and a first-party `AddJwtBearer` scheme (`Bearer`, +> `ValidAlgorithms = HS256`, parameters from `JwtTokenService.BuildValidationParameters`) +> accepts it on `/api/*` and every `RequireAuthorization` endpoint through the +> `OtOpcUa.CookieOrBearer` default policy scheme. Without a key neither exists. DataProtection > keys persist to the shared Config DB (`PersistKeysToDbContext`) > so cookies survive failover between admin-role nodes. > diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs deleted file mode 100644 index cc996758..00000000 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs +++ /dev/null @@ -1,99 +0,0 @@ -using System.Text.Encodings.Web; -using Microsoft.AspNetCore.Authentication; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Primitives; -using ZB.MOM.WW.OtOpcUa.Security.Jwt; - -namespace ZB.MOM.WW.OtOpcUa.Security.Auth; - -/// The authentication scheme names OtOpcUa registers. -public static class OtOpcUaAuthSchemes -{ - /// - /// The bearer scheme that validates the HS256 tokens /auth/token mints, through - /// . Registered only when - /// . - /// - public const string Bearer = "Bearer"; - - /// - /// The default scheme: a policy scheme that forwards to when the request - /// carries an Authorization: Bearer header and to the cookie scheme otherwise, so one - /// RequireAuthorization() / policy accepts a browser session AND a machine token. - /// - public const string CookieOrBearer = "OtOpcUa.CookieOrBearer"; - - /// Whether the request presents a bearer token (the selector's whole rule). - /// The request. - /// when an Authorization: Bearer … header is present. - public static bool HasBearerToken(HttpContext context) => - TryGetBearerToken(context.Request.Headers.Authorization, out _); - - internal static bool TryGetBearerToken(StringValues authorization, out string token) - { - token = string.Empty; - var header = authorization.ToString(); - const string prefix = "Bearer "; - if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; - token = header[prefix.Length..].Trim(); - return token.Length > 0; - } -} - -/// -/// Validates an Authorization: Bearer token minted by /auth/token (R03:2.4). The JWT -/// feature used to be a dead end: tokens were issued and nothing in the product accepted them. This -/// handler is the consuming side, built on so the validation -/// parameters can never drift from the minting side ( -/// sets RoleClaimType/NameClaimType to the minted keys, so [Authorize(Roles)] and -/// IsInRole resolve). -/// -/// -/// Fail closed: no header ⇒ NoResult (the cookie scheme is not consulted by this handler — the -/// policy scheme selects one or the other); a malformed, expired, tampered or wrong-issuer token ⇒ -/// Fail, never a fall-through to the cookie. A direct AddJwtBearer needs the -/// Microsoft.AspNetCore.Authentication.JwtBearer package, which the central package list does -/// not carry; this handler is ~40 lines over the already-referenced System.IdentityModel.Tokens.Jwt -/// and is a drop-in to replace once that package is pinned. -/// -public sealed class JwtBearerAuthenticationHandler : AuthenticationHandler -{ - private readonly JwtTokenService _jwt; - - /// Initializes a new instance of the class. - /// The scheme options monitor. - /// The logger factory. - /// The URL encoder. - /// The token service whose validation parameters mirror its minting side. - public JwtBearerAuthenticationHandler( - IOptionsMonitor options, - ILoggerFactory logger, - UrlEncoder encoder, - JwtTokenService jwt) - : base(options, logger, encoder) - { - _jwt = jwt; - } - - /// - protected override Task HandleAuthenticateAsync() - { - if (!OtOpcUaAuthSchemes.TryGetBearerToken(Request.Headers.Authorization, out var token)) - return Task.FromResult(AuthenticateResult.NoResult()); - - if (!_jwt.TryValidate(token, out var principal) || principal is null) - return Task.FromResult(AuthenticateResult.Fail("Invalid bearer token.")); - - return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(principal, Scheme.Name))); - } - - /// - protected override Task HandleChallengeAsync(AuthenticationProperties properties) - { - Response.StatusCode = StatusCodes.Status401Unauthorized; - Response.Headers.WWWAuthenticate = OtOpcUaAuthSchemes.Bearer; - return Task.CompletedTask; - } -} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs new file mode 100644 index 00000000..7c30915e --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs @@ -0,0 +1,39 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; + +namespace ZB.MOM.WW.OtOpcUa.Security.Auth; + +/// The authentication scheme names OtOpcUa registers. +public static class OtOpcUaAuthSchemes +{ + /// + /// The bearer scheme that validates the HS256 tokens /auth/token mints: the first-party + /// AddJwtBearer handler over . + /// Registered only when . + /// + public const string Bearer = "Bearer"; + + /// + /// The default scheme: a policy scheme that forwards to when the request + /// carries an Authorization: Bearer header and to the cookie scheme otherwise, so one + /// RequireAuthorization() / policy accepts a browser session AND a machine token. + /// + public const string CookieOrBearer = "OtOpcUa.CookieOrBearer"; + + /// Whether the request presents a bearer token (the selector's whole rule). + /// The request. + /// when an Authorization: Bearer … header is present. + public static bool HasBearerToken(HttpContext context) => + TryGetBearerToken(context.Request.Headers.Authorization, out _); + + internal static bool TryGetBearerToken(StringValues authorization, out string token) + { + token = string.Empty; + var header = authorization.ToString(); + const string prefix = "Bearer "; + if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; + token = header[prefix.Length..].Trim(); + return token.Length > 0; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs index 6b0876d7..a3f661b6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs @@ -24,6 +24,17 @@ public static class AuthEndpoints /// Response for a token issue request. public sealed record TokenResponse(string Token); + /// + /// The /auth/token policy: an authenticated principal from the COOKIE scheme. Naming the scheme + /// makes the authorization middleware authenticate that scheme alone, so an Authorization: Bearer + /// header is never consulted here (with login disabled the auto-login handler sits under the same + /// scheme name, so the dev rig is unchanged). + /// + internal static readonly AuthorizationPolicy CookieSessionOnly = + new AuthorizationPolicyBuilder(CookieAuthenticationDefaults.AuthenticationScheme) + .RequireAuthenticatedUser() + .Build(); + /// Maps OtOpcUa authentication endpoints to the application route builder. /// The endpoint route builder. /// The endpoint route builder for chaining. @@ -40,7 +51,10 @@ public static class AuthEndpoints // Mapped only when a signing key is configured (JwtOptions.IsConfigured): an unconfigured // endpoint used to answer HTTP 500 from the JwtTokenService constructor on every call. if (app.ServiceProvider.GetRequiredService>().Value.IsConfigured) - app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(); + // COOKIE scheme only (fix round 1 / I2): the default scheme also accepts a bearer, and a bearer + // that could mint a fresh token would renew itself forever — outliving its own expiry and the + // user's LDAP membership. A token is minted only for an interactive (cookie) session. + app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(CookieSessionOnly); else app.ServiceProvider.GetService()? .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs index 4dd38a2e..9dae8c14 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs @@ -25,24 +25,12 @@ public sealed class JwtTokenService /// /// Role claim type used in the JWT payload. /// - /// Issued-only / no internal JwtBearer scheme: OtOpcUa uses a single Cookie - /// authentication scheme; the JWT is minted by the /auth/token endpoint and - /// consumed externally (e.g. by OPC-UA clients or automation scripts). There is no - /// AddJwtBearer pipeline in OtOpcUa — the cookie stores the - /// directly. Because no internal - /// bearer validation path exists, the short "Role" key is intentionally used here rather - /// than the long URI; external consumers receive exactly the - /// key they expect. - /// - /// - /// If a JwtBearer scheme is ever added: the - /// passed to - /// AddJwtBearer MUST set RoleClaimType = JwtTokenService.RoleClaimType (and - /// NameClaimType = JwtTokenService.UsernameClaimType) so that - /// [Authorize(Roles=...)] and ClaimsPrincipal.IsInRole resolve correctly. - /// is already wired to do this and MUST be used - /// rather than constructing - /// ad hoc. + /// The short "Role" key is what mints and what external consumers of the token + /// expect. The internal bearer scheme (AddJwtBearer in AddOtOpcUaAuth) validates with + /// , which sets RoleClaimType = RoleClaimType and + /// NameClaimType = UsernameClaimType, and turns MapInboundClaims off, so + /// [Authorize(Roles=...)] and ClaimsPrincipal.IsInRole resolve against this key. Never + /// construct ad hoc. /// /// public const string RoleClaimType = "Role"; @@ -150,5 +138,9 @@ public sealed class JwtTokenService // if this is ever passed to AddJwtBearer. See RoleClaimType doc comment for rationale. RoleClaimType = RoleClaimType, NameClaimType = UsernameClaimType, + // Pinned to the one algorithm Issue signs with: an unsigned (alg:none) token or one signed with a + // different HMAC variant over the same key is refused on both the bearer and TryValidate paths. + ValidAlgorithms = [SecurityAlgorithms.HmacSha256], + RequireSignedTokens = true, }; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs index d58fbcf3..494145be 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.Cookies; +using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; @@ -108,8 +109,21 @@ public static class ServiceCollectionExtensions }); if (jwtConfigured) { - authBuilder.AddScheme( - OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { }); + // First-party JwtBearer (R03:2.4, fix round 1 / I1) — the earlier hand-rolled handler is gone. + // Validation parameters come from the SAME JwtTokenService that mints, so issuer/audience/key/ + // claim types can never drift; BuildValidationParameters pins ValidAlgorithms = HS256 (an + // alg:none or a different-HMAC token is refused). MapInboundClaims=false keeps the minted + // "Role"/"zb:username" claim keys verbatim, which RoleClaimType/NameClaimType name. A bad token + // is AuthenticateResult.Fail ⇒ 401 + WWW-Authenticate: Bearer, never a 500. + authBuilder.AddJwtBearer(OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { }); + services.AddOptions(OtOpcUaAuthSchemes.Bearer) + .Configure((o, jwt) => + { + o.TokenValidationParameters = jwt.BuildValidationParameters(); + o.MapInboundClaims = false; + o.SaveToken = false; + o.IncludeErrorDetails = false; + }); } if (disableLogin) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj index d81e4eb6..507aca6c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj @@ -10,6 +10,7 @@ + diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs index e5e3b620..599d0cf2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs @@ -12,6 +12,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; using Shouldly; using Xunit; using ZB.MOM.WW.Auth.AspNetCore; @@ -208,6 +209,98 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); } + /// + /// Fix round 1 / I1 — the first-party JwtBearer handler pins HS256: an unsigned (alg: none) + /// token carrying the right issuer/audience/roles, and a token signed with a different HMAC + /// variant, both fail CLOSED with a 401 — never a 500, never a pass. + /// + [Theory] + [InlineData("none")] + [InlineData("HS384")] + public async Task Bearer_with_a_non_HS256_algorithm_is_rejected_with_401(string alg) + { + var token = alg == "none" ? UnsignedToken() : SignedToken(SecurityAlgorithms.HmacSha384); + var client = NewClient(); + var req = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + + var response = await client.SendAsync(req, Ct); + + response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized); + response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); + + // Positive control: the same claims signed HS256 with the configured key ARE accepted, so the + // 401 above measured the algorithm and not a broken token shape. + var good = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + good.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue( + "Bearer", SignedToken(SecurityAlgorithms.HmacSha256)); + (await client.SendAsync(good, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + } + + /// + /// Fix round 1 / I2 — /auth/token mints only for a COOKIE session. A bearer token used to + /// be able to mint a fresh token for itself, so a stolen token could renew itself forever and + /// outlive both its own expiry and the user's LDAP membership. Positive control: the cookie + /// session that minted the bearer still gets a token. + /// + [Fact] + public async Task Auth_token_refuses_a_bearer_only_caller_and_serves_a_cookie_session() + { + var client = NewClient(); + var loginResponse = await client.PostAsJsonAsync("/auth/login", + new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + loginResponse.EnsureSuccessStatusCode(); + + var cookieReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + AttachCookies(cookieReq, loginResponse); + var cookieResp = await client.SendAsync(cookieReq, Ct); + cookieResp.StatusCode.ShouldBe(HttpStatusCode.OK); + var bearer = (await cookieResp.Content.ReadFromJsonAsync(Ct)).GetProperty("token").GetString()!; + + // The bearer authenticates elsewhere (positive control on the token itself) … + var root = new HttpRequestMessage(HttpMethod.Get, "/"); + root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer); + (await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + // … but cannot renew itself. + var renew = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + renew.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer); + renew.Headers.Add("X-Requested-With", "XMLHttpRequest"); + var renewResp = await client.SendAsync(renew, Ct); + renewResp.StatusCode.ShouldBeOneOf(HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden); + (await renewResp.Content.ReadAsStringAsync(Ct)).ShouldNotContain("token"); + } + + private const string TestSigningKey = "test-signing-key-with-at-least-32-bytes-of-utf8-content"; + + private static IEnumerable ViewerClaims() => + [ + new(JwtTokenService.UsernameClaimType, "mallory"), + new(JwtTokenService.RoleClaimType, "Viewer"), + ]; + + /// A token signed with using the configured key bytes. + private static string SignedToken(string algorithm) + { + var keyBytes = System.Text.Encoding.UTF8.GetBytes(TestSigningKey); + // The SAME key bytes for every algorithm, so a rejection measures the algorithm pin alone. + var token = new System.IdentityModel.Tokens.Jwt.JwtSecurityToken( + issuer: "otopcua-test", audience: "otopcua-test", claims: ViewerClaims(), + expires: DateTime.UtcNow.AddMinutes(5), + signingCredentials: new SigningCredentials(new SymmetricSecurityKey(keyBytes), algorithm)); + return new System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler().WriteToken(token); + } + + /// An alg: none token with otherwise valid issuer/audience/lifetime/roles. + private static string UnsignedToken() + { + static string B64(string json) => Microsoft.IdentityModel.Tokens.Base64UrlEncoder.Encode(json); + var exp = DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeSeconds(); + return B64("{\"alg\":\"none\",\"typ\":\"JWT\"}") + "." + + B64($"{{\"zb:username\":\"mallory\",\"Role\":\"Viewer\",\"iss\":\"otopcua-test\",\"aud\":\"otopcua-test\",\"exp\":{exp}}}") + + "."; + } + private async Task LoginAndIssueTokenAsync() { var client = NewClient(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs index 96fc88ca..9d2cc7ef 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs @@ -65,7 +65,7 @@ public sealed class JwtConfigurationGateTests var schemes = host.Services.GetRequiredService(); var bearer = await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer); bearer.ShouldNotBeNull(); - bearer.HandlerType.ShouldBe(typeof(JwtBearerAuthenticationHandler)); + bearer.HandlerType.ShouldBe(typeof(Microsoft.AspNetCore.Authentication.JwtBearer.JwtBearerHandler)); await host.StopAsync(Ct); } From c0a0e7aa81626b5ff31e4a75284ac0aa5205334d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:36:51 -0400 Subject: [PATCH 17/27] fix(security): audit actor falls back to HttpContext when the server auth-state provider has no circuit ServerAuthenticationStateProvider throws InvalidOperationException from GetAuthenticationStateAsync in any scope where no circuit set its state (minimal-API requests, hosted-service scopes). The accessor now treats that as "no circuit identity" and falls back to the HTTP principal. Pinned with the real provider on both sides (unset -> HTTP fallback; set -> circuit principal wins). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AuthenticationStateAuditActorAccessor.cs | 32 +++++++++++++---- ...henticationStateAuditActorAccessorTests.cs | 36 +++++++++++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs index d689b85e..13ede448 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs @@ -47,17 +47,37 @@ public sealed class AuthenticationStateAuditActorAccessor : IAuditActorAccessor { if (_authenticationState is not null) { - var task = _authenticationState.GetAuthenticationStateAsync(); - if (task.IsCompletedSuccessfully) - { - var actor = ResolveActor(task.Result.User); - if (actor is not null) return actor; - } + var actor = TryResolveCircuitActor(_authenticationState); + if (actor is not null) return actor; } return ResolveActor(_httpContextAccessor.HttpContext?.User); } } + /// + /// The circuit's actor, or null when this scope has no circuit identity. The server-side provider + /// is registered for EVERY scope on an AdminUI node, but only a circuit calls + /// SetAuthenticationState; anywhere else (a minimal-API request, a hosted service's scope) + /// GetAuthenticationStateAsync throws synchronously + /// (fix round 1 / I5). That is "no circuit", never an audit failure — the caller falls back to + /// the HTTP context. + /// + /// The scoped provider. + /// The actor, or null. + private static string? TryResolveCircuitActor(AuthenticationStateProvider provider) + { + Task task; + try + { + task = provider.GetAuthenticationStateAsync(); + } + catch (InvalidOperationException) + { + return null; + } + return task.IsCompletedSuccessfully ? ResolveActor(task.Result.User) : null; + } + /// The claim-resolution rule shared with . /// The principal, or null. /// The actor string, or null when unauthenticated. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs index 5dc1f391..6341ef6f 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs @@ -58,6 +58,42 @@ public sealed class AuthenticationStateAuditActorAccessorTests accessor.CurrentActor.ShouldBeNull(); } + /// + /// Fix round 1 / I5 — the REAL server-side provider, resolved in a non-circuit scope (a minimal-API + /// request, a background service), throws from + /// GetAuthenticationStateAsync because nothing called SetAuthenticationState. That used + /// to escape CurrentActor; it must fall back to the HTTP principal instead. + /// + [Fact] + public void Real_server_provider_outside_a_circuit_falls_back_to_the_http_principal() + { + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "dave")) }; + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(context), + new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider()); + + accessor.CurrentActor.ShouldBe("dave"); + + // No HTTP principal either ⇒ null, still no throw. + new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider()) + .CurrentActor.ShouldBeNull(); + } + + /// Positive control on the real provider: once a circuit has set its state, that principal wins. + [Fact] + public void Real_server_provider_inside_a_circuit_serves_the_circuit_principal() + { + var provider = new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider(); + provider.SetAuthenticationState(Task.FromResult( + new AuthenticationState(Authenticated(new Claim(ZbClaimTypes.Username, "erin"))))); + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "http-user")) }; + + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), provider) + .CurrentActor.ShouldBe("erin"); + } + private static ClaimsPrincipal Authenticated(params Claim[] claims) => new(new ClaimsIdentity(claims, "TestScheme", ZbClaimTypes.Name, ZbClaimTypes.Role)); From 88268ea3a6980a0c473d18b3acc8740add65d426 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:38:03 -0400 Subject: [PATCH 18/27] fix(host): OPC UA endpoint probe is Ready-only on a fused admin+driver node Coordinator ruling (I6): until ServiceLevel consumes IOpcUaEndpointState (F-C1), the probe must not sit on the Active tier of a fused node, because Traefik routes the AdminUI on /health/active. Driver-only nodes keep it on Ready + Active; admin-only nodes do not register it. Pinned per role. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Health/HealthEndpoints.cs | 21 ++++++++++++---- .../Health/OpcUaEndpointHealthCheck.cs | 6 +++-- .../Health/ActiveTierRegistrationTests.cs | 25 ++++++++++++------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs index 2f57158f..824223dd 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs @@ -31,7 +31,8 @@ public static class HealthEndpoints /// /// /// Whether this node carries the driver role and therefore serves an OPC UA endpoint. When - /// true the joins the ready + active tiers (R03:2.1), reading + /// true the joins the ready tier, and the active tier on a + /// driver-ONLY node (R03:2.1; on a fused node the active tier routes the AdminUI — see the body), reading /// the the host registers alongside OtOpcUaServerHostedService. /// Defaults to false so admin-only callers and the existing harnesses register nothing new. /// @@ -40,16 +41,26 @@ public static class HealthEndpoints { var checks = services.AddHealthChecks(); - // A driver node's whole purpose is its OPC UA endpoint, so a failed SDK start must take it out of - // the ready AND active tiers: readiness because it cannot serve, active because a load balancer or - // the fleet must not route to it. Factory-registered so the state singleton is resolved per probe. + // A failed SDK start takes the node out of the READY tier on every driver node: it cannot serve. + // The ACTIVE tier depends on the roles (coordinator ruling, fix round 1 / I6): + // - driver-only: Active too — a site node's whole purpose is its OPC UA endpoint, so nothing may + // route to it; + // - fused admin+driver: NOT Active, until the ServiceLevel calculator consumes IOpcUaEndpointState + // (follow-up F-C1). Traefik routes the AdminUI on /health/active, so putting the probe there would + // take the operator's only console out of rotation precisely when they need it to diagnose the + // failed endpoint, while the OPC UA failover (ServiceLevel) still would not move. Once F-C1 lands + // the peer takes Primary and the active tier follows it on its own. + // Factory-registered so the state singleton is resolved per probe. if (hasDriver) { + var tags = hasAdmin + ? new[] { ZbHealthTags.Ready } + : new[] { ZbHealthTags.Ready, ZbHealthTags.Active }; checks.Add(new HealthCheckRegistration( OpcUaEndpointHealthCheck.Name, sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService()), failureStatus: null, - tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })); + tags: tags)); } // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs index 8caa69f5..e619d7b3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs @@ -4,14 +4,16 @@ using ZB.MOM.WW.OtOpcUa.Host.OpcUa; namespace ZB.MOM.WW.OtOpcUa.Host.Health; /// -/// Ready + Active tier probe for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is +/// Ready-tier probe (plus Active on a driver-only node) for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is /// listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy /// one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets /// connection-refused. /// /// /// Registered on driver-role nodes only (); an -/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. +/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. On a fused +/// admin+driver node it is Ready-only until ServiceLevel consumes the fact (F-C1), because the Active tier +/// is what Traefik routes the AdminUI on. /// public sealed class OpcUaEndpointHealthCheck : IHealthCheck { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs index d880e7b3..45ff262b 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs @@ -50,31 +50,38 @@ public sealed class ActiveTierRegistrationTests } /// - /// R03:2.1 — the OPC UA endpoint probe is on BOTH tiers of a driver node (it cannot serve, and must not - /// be routed to) and absent from an admin-only node, which starts no endpoint to be degraded by. + /// R03:2.1 + fix round 1 / I6 (coordinator ruling) — the OPC UA endpoint probe's tiers by role: + /// + /// driver-only: Ready + Active (it cannot serve and must not be routed to); + /// fused admin+driver: Ready ONLY, until the ServiceLevel calculator consumes the fact + /// (F-C1) — Traefik routes the AdminUI on /health/active, so a failed SDK on the active + /// central node must not take the AdminUI out of rotation; + /// admin-only: absent (it starts no endpoint to be degraded by). + /// /// [Theory] - [InlineData(true)] - [InlineData(false)] - public void Opcua_endpoint_probe_is_registered_on_ready_and_active_for_driver_nodes_only(bool hasDriver) + [InlineData(false, true, true, true)] + [InlineData(true, true, true, false)] + [InlineData(true, false, false, false)] + public void Opcua_endpoint_probe_tiers_follow_the_node_roles(bool hasAdmin, bool hasDriver, bool ready, bool active) { var registrations = new ServiceCollection() - .AddOtOpcUaHealth(hasAdmin: true, hasDriver: hasDriver) + .AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: hasDriver) .BuildServiceProvider() .GetRequiredService>() .Value.Registrations .Where(r => r.Name == OpcUaEndpointHealthCheck.Name) .ToList(); - if (!hasDriver) + if (!ready && !active) { registrations.ShouldBeEmpty(); return; } var registration = registrations.ShouldHaveSingleItem(); - registration.Tags.ShouldContain(ZbHealthTags.Ready); - registration.Tags.ShouldContain(ZbHealthTags.Active); + registration.Tags.Contains(ZbHealthTags.Ready).ShouldBe(ready); + registration.Tags.Contains(ZbHealthTags.Active).ShouldBe(active); } [Fact] From f1c52385edf42387e892a00dcad64bfb133a7838 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:39:34 -0400 Subject: [PATCH 19/27] fix(controlplane): retention keep window counts Sealed rows; the newest Sealed blob is never pruned The window used to count every terminal row, so 50+ failed deploys pushed the last good artifact out of it. Now the newest KeepSealed (>= 1) Sealed rows keep their blob, failed rows are prunable unless referenced in the ReferencedDays window. Pinned: 60 TimedOut + 1 old Sealed -> Sealed retained, 60 TimedOut emptied. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Retention/DeploymentRetentionActor.cs | 21 ++++-- .../Retention/DeploymentRetentionOptions.cs | 7 +- .../DeploymentRetentionTests.cs | 65 ++++++++++++++++++- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs index 55ffd7dc..67020803 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs @@ -97,8 +97,9 @@ public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers } /// - /// The retention rule. Empties the blob of every terminal deployment that is (a) older than the - /// newest terminal rows, (b) not referenced by a + /// The retention rule. Empties the blob of every terminal deployment that is (a) not one of the + /// newest SEALED rows (at least one — the newest + /// Sealed blob is never pruned), (b) not referenced by a /// NodeDeploymentState row started or applied within /// , and (c) still carrying bytes — at most /// per call. @@ -125,11 +126,23 @@ public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers || d.Status == DeploymentStatus.PartiallyFailed || d.Status == DeploymentStatus.TimedOut) .OrderByDescending(d => d.CreatedAtUtc) - .Select(d => new { d.DeploymentId, HasBlob = d.ArtifactBlob.Length > 0 }) + .Select(d => new { d.DeploymentId, d.Status, HasBlob = d.ArtifactBlob.Length > 0 }) .ToListAsync(ct) .ConfigureAwait(false); - var aged = terminal.Skip(keep).Where(d => d.HasBlob).Select(d => d.DeploymentId).ToList(); + // The keep window counts SEALED rows only (fix round 1 / I3). Counting every terminal row let a burst + // of failed deploys (TimedOut / PartiallyFailed) fill the window by itself and push the fleet's last + // good artifact — the newest Sealed row, which boot, drift detection and the publish fallback all + // read — out of it. `keep` is clamped to ≥ 1, so the newest Sealed row is never a candidate. + var keptSealed = terminal + .Where(d => d.Status == DeploymentStatus.Sealed) + .Take(keep) + .Select(d => d.DeploymentId) + .ToHashSet(); + var aged = terminal + .Where(d => d.HasBlob && !keptSealed.Contains(d.DeploymentId)) + .Select(d => d.DeploymentId) + .ToList(); if (aged.Count == 0) return 0; var referenced = (await db.NodeDeploymentStates diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs index 6654ec1e..6bdc571f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs @@ -17,9 +17,10 @@ public sealed class DeploymentRetentionOptions public bool Enabled { get; set; } = true; /// - /// How many of the most recent terminal (Sealed / PartiallyFailed / TimedOut) deployments keep their - /// blob regardless of age. A node booting from central SQL reads the LATEST sealed row's blob, so this - /// must stay well above 1; 50 also keeps a comfortable re-deploy history. + /// How many of the most recent SEALED deployments keep their blob regardless of age (clamped to ≥ 1). + /// Failed deploys (PartiallyFailed / TimedOut) do not count toward it, so a burst of them can never push + /// the last good artifact out. A node booting from central SQL reads the LATEST sealed row's blob, so + /// this must stay well above 1; 50 also keeps a comfortable re-deploy history. /// public int KeepSealed { get; set; } = 50; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs index ecd397c4..bb607aaf 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs @@ -24,7 +24,7 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase public async Task Sweep_empties_blobs_beyond_the_keep_window_and_leaves_the_newest_intact() { var f = NewInMemoryDbFactory(); - var ids = Seed(f, sealedCount: 12); // ids[0] newest … ids[11] oldest + var ids = Seed(f, sealedCount: 12, mixInTimedOut: false); // ids[0] newest … ids[11] oldest var options = new DeploymentRetentionOptions { KeepSealed = 5, ReferencedDays = 30, MaxRowsPerSweep = 200 }; int pruned; @@ -91,6 +91,65 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase (await verify.Deployments.SingleAsync(d => d.DeploymentId == inFlight)).ArtifactBlob.ShouldNotBeEmpty("in flight"); } + /// + /// Fix round 1 / I3 — the keep window counts SEALED rows, not "terminal" ones. A burst of 60 failed + /// deploys used to fill a 50-row terminal window on its own and push the fleet's last good (Sealed) + /// artifact out of it — the one every boot / drift / fallback path reads. The newest Sealed blob is + /// never pruned; the unreferenced TimedOut rows are (positive control: the sweep did run). + /// + [Fact] + public async Task Sixty_timed_out_deploys_never_prune_the_newest_sealed_blob() + { + var f = NewInMemoryDbFactory(); + var sealedId = Guid.NewGuid(); + var timedOut = new List(); + await using (var db = f.CreateDbContext()) + { + db.Deployments.Add(new Deployment + { + DeploymentId = sealedId, RevisionHash = new string('s', 64), Status = DeploymentStatus.Sealed, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-200), SealedAtUtc = SweepNow.AddDays(-200), + ArtifactBlob = new byte[] { 1, 2, 3 }, + }); + for (var i = 0; i < 60; i++) + { + var id = Guid.NewGuid(); + timedOut.Add(id); + db.Deployments.Add(new Deployment + { + DeploymentId = id, RevisionHash = new string('t', 64), Status = DeploymentStatus.TimedOut, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddHours(-i), ArtifactBlob = new byte[] { 9 }, + }); + } + await db.SaveChangesAsync(); + } + var options = new DeploymentRetentionOptions { KeepSealed = 50, ReferencedDays = 30, MaxRowsPerSweep = 200 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(60); + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == sealedId)).ArtifactBlob + .ShouldNotBeEmpty("the newest Sealed artifact is never pruned"); + (await verify.Deployments.CountAsync(d => timedOut.Contains(d.DeploymentId) && d.ArtifactBlob.Length == 0)) + .ShouldBe(60); + } + + /// Even with KeepSealed misconfigured to 0, the newest Sealed row keeps its blob. + [Fact] + public async Task The_newest_sealed_blob_survives_a_zero_keep_window() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 3, mixInTimedOut: false); + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, new DeploymentRetentionOptions { KeepSealed = 0 }, SweepNow)) + .ShouldBe(2); + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty(); + } + [Fact] public async Task Sweep_is_bounded_per_call_and_drains_a_backlog_over_calls() { @@ -149,7 +208,7 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase } /// Seeds sealed deployments, newest first; each one hour older than the previous. - private static List Seed(IDbContextFactory f, int sealedCount) + private static List Seed(IDbContextFactory f, int sealedCount, bool mixInTimedOut = true) { var ids = new List(); using var db = f.CreateDbContext(); @@ -161,7 +220,7 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase { DeploymentId = id, RevisionHash = new string('a', 64), - Status = i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed, + Status = mixInTimedOut && i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed, CreatedBy = "test", CreatedAtUtc = SweepNow.AddHours(-i), SealedAtUtc = SweepNow.AddHours(-i), From ef2eebe6b6c46dcdc5bde3afad20cb44beeed08a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:41:15 -0400 Subject: [PATCH 20/27] fix(scripting): review minors: configured limit in summaries, dispose gate/CTS, validator, docs (fix round 1) - ScriptLogRateLimiter.LimitPerWindow. The ScriptLogTopicSink and VirtualTagActor summaries now print the configured limit, not DefaultLimitPerWindow. Tests assert "limit N/window" for a non-default N. - CalculationDriver.Dispose cancels, then takes _evalGate (bounded 2 s) and disposes the gate and _evalCts. The pump and timer ticks enter the gate through TryEnterEvalGateAsync, which stops cleanly once the driver is disposed. Test: Dispose with a 6 s runaway run in flight returns promptly, disposes both, and a late trigger is harmless. - ScriptedAlarmEngine captures _shutdownToken once and disposes _shutdownCts in Dispose when every drained task has completed. A task stuck past the grace leaves the CTS to the GC. Lifecycle tests assert the disposal. - DraftValidator: CalculationNoEffectiveTrigger is skipped for a change-triggered tag whose scriptId does not resolve. That case is already CalculationScriptMissing or CalculationScriptNotFound, and there are no reads to count. Both existing tests now assert the absence (red before the change). - docs/VirtualTags.md: System.Threading is denied as a whole namespace, with an allow-list; the type list is now accurate (Unsafe, RuntimeHelpers, and the ForbiddenMembers on System.Type); the resource limits section covers the run guard and the allocation budget. docs/ScriptEditor.md: accurate deny-list and a new "Runtime safety" section covering timeout, refusal, quarantine after 3 timeouts, allocation budget, lifting, and rate-limit summaries that appear only on the source's next entry after a burst then quiet. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/ScriptEditor.md | 42 +++++++++++++++- docs/VirtualTags.md | 8 +-- .../Validation/DraftValidator.cs | 4 ++ .../ScriptedAlarmEngine.cs | 28 +++++++++-- .../ScriptLogRateLimiter.cs | 3 ++ .../ScriptLogTopicSink.cs | 2 +- .../CalculationDriver.cs | 49 +++++++++++++++++-- .../VirtualTags/VirtualTagActor.cs | 2 +- .../DraftValidatorCalculationTests.cs | 10 +++- .../ScriptedAlarmEngineLifecycleTests.cs | 3 ++ .../ScriptLogRateLimiterTests.cs | 4 ++ .../CalculationDriverRemediationTests.cs | 23 +++++++++ ...VirtualTagActorFailureLogRateLimitTests.cs | 1 + 13 files changed, 161 insertions(+), 18 deletions(-) diff --git a/docs/ScriptEditor.md b/docs/ScriptEditor.md index 5321aca8..41421d34 100644 --- a/docs/ScriptEditor.md +++ b/docs/ScriptEditor.md @@ -162,11 +162,49 @@ user's fault. `ForbiddenTypeAnalyzer.Analyze(compilation)` walks the wrapped syntax tree and resolves every referenced symbol against the sandbox deny-list (namespace prefixes `System.IO`, `System.Net`, `System.Diagnostics`, `System.Reflection`, -`System.Threading.Tasks`, etc., plus type-granular denials like -`System.Environment` and `System.Threading.Thread`). Each violation is mapped +`System.Threading` — the whole namespace, with only `CancellationToken`, +`Interlocked` and `Volatile` allowed back in — `System.Threading.Tasks`, +`System.Runtime.InteropServices`, `System.Runtime.Loader`, `Microsoft.Win32`; +type-granular denials `System.Environment`, `System.AppDomain`, `System.GC`, +`System.Activator`, `System.Runtime.CompilerServices.Unsafe` / +`RuntimeHelpers`; and member-granular denials on `System.Type`'s by-name +reflection entry points — `GetType(string)`, `GetMethod(s)`, `InvokeMember`, +and the `GetProperty`/`GetField`/`GetMember`/`GetConstructor`/`GetEvent` +families). The runtime publish gate uses the same analyzer, so a deny-list +change needs a `DiagnoseTests` case to keep editor ⇔ publish parity. Each violation is mapped via `tree.GetMappedLineSpan(span)` to user-source coordinates and emitted as an error marker with code `OTSCRIPT_FORBIDDEN`. +### Runtime safety (what happens after publish) + +The editor cannot catch a script that compiles cleanly but never returns or +allocates without bound, so every runtime seam (virtual tags, Calculation tags, +scripted-alarm predicates) runs scripts through the shared `ScriptRunGuard`: + +- **Timeout + one orphan.** A run that exceeds its budget is abandoned (it cannot + be pre-empted in-process) and the tag/alarm reads Bad. While that abandoned run + is still executing, every further trigger is **refused** — Bad + `"previous evaluation still running"`, counted on `otopcua.script.run_refused` — + so a `while(true)` costs one pool thread, not one per trigger. +- **Quarantine after 3 consecutive timeouts** (`Scripting:QuarantineAfterConsecutiveTimeouts`). + Only a timeout is a strike: a refusal neither adds to nor resets the streak, and + a run that returns (value or throw) resets it. A quarantined script is never + dispatched again; it reads Bad `"script quarantined: …"`, logs one Error and + increments `otopcua_script_quarantined`. An infinite loop therefore usually shows + as *refused* (one timeout, then refusals for as long as the orphan spins) rather + than *quarantined*. +- **Allocation budget.** A single run allocating more than + `Scripting:MaxAllocBytesPerRun` (64 MB) is quarantined. This is sampled after the + run returns, so a run that OOMs the process first is not stopped — that needs an + out-of-process runner. +- **Lifting.** A quarantine lifts when the script source changes or on redeploy, + never by itself. +- **Script-log rate limit.** The cluster-wide script-log fan-out admits 20 entries + per source per minute; the rest are counted and replaced by one "N suppressed" + summary. The summary is emitted on that source's **next** entry after the window + rolls, so a source that bursts and then goes quiet shows its summary only when + it logs again (the full stream is always in `scripts-*.log`). + ### 3. DependencyExtractor `DependencyExtractor.Extract(code).Rejections` enumerates every `ctx.GetTag` / diff --git a/docs/VirtualTags.md b/docs/VirtualTags.md index d58013ea..50ef578f 100644 --- a/docs/VirtualTags.md +++ b/docs/VirtualTags.md @@ -28,15 +28,15 @@ User scripts are compiled via `Microsoft.CodeAnalysis.CSharp` (regular compiler, `ScriptSandbox.Build` constructs the compile reference set in two parts. First, four pinned OtOpcUa assemblies are always included: `Core.Abstractions` (for `DataValueSnapshot` / `DriverDataType`), `Core.Scripting` (for `ScriptContext` + `Deadband`), `Serilog` (for `ILogger`), and the concrete context type's assembly. Second, the BCL subset is enumerated from the runtime's `TRUSTED_PLATFORM_ASSEMBLIES` list, restricted to filenames starting with `System.*` plus `netstandard.dll`, `mscorlib.dll`, and `Microsoft.Win32.Registry.dll` (the last needed so `ForbiddenTypeAnalyzer` can resolve and reject registry types). Pre-imported namespaces: `System`, `System.Linq`, `ZB.MOM.WW.OtOpcUa.Core.Abstractions`, `ZB.MOM.WW.OtOpcUa.Core.Scripting`. -`ForbiddenTypeAnalyzer.ForbiddenNamespacePrefixes` denies `System.IO`, `System.Net`, `System.Diagnostics`, `System.Reflection`, `System.Threading.Tasks`, `System.Runtime.InteropServices`, `System.Runtime.Loader`, and `Microsoft.Win32`. Matching is by prefix against the resolved symbol's containing namespace, so `System.Net` catches `System.Net.Http.HttpClient` and every subnamespace. `System.Threading.Tasks` is denied because scripts are synchronous predicates with no legitimate need to start background tasks — a `Task.Run` fan-out would outlive the per-evaluation timeout entirely (Core.Scripting-003). `System.Runtime.Loader` is denied to block `AssemblyLoadContext` / `AssemblyDependencyResolver` — arbitrary DLL loads into the host process (Core.Scripting-012). +`ForbiddenTypeAnalyzer.ForbiddenNamespacePrefixes` denies `System.IO`, `System.Net`, `System.Diagnostics`, `System.Reflection`, `System.Threading` (the whole namespace — `ForbiddenTypeAnalyzer.AllowedTypesInForbiddenNamespaces` lets only `CancellationToken`, `Interlocked` and `Volatile` back in), `System.Threading.Tasks`, `System.Runtime.InteropServices`, `System.Runtime.Loader`, and `Microsoft.Win32`. Matching is by prefix against the resolved symbol's containing namespace, so `System.Net` catches `System.Net.Http.HttpClient` and every subnamespace. `System.Threading.Tasks` is denied because scripts are synchronous predicates with no legitimate need to start background tasks — a `Task.Run` fan-out would outlive the per-evaluation timeout entirely (Core.Scripting-003). `System.Runtime.Loader` is denied to block `AssemblyLoadContext` / `AssemblyDependencyResolver` — arbitrary DLL loads into the host process (Core.Scripting-012). -`ForbiddenTypeAnalyzer.ForbiddenFullTypeNames` denies type-granularly: `System.Environment`, `System.AppDomain`, `System.GC`, `System.Activator`, `System.Threading.Thread`, `System.Threading.ThreadPool`, and `System.Threading.Timer`. These types require granular denial rather than namespace-prefix denial for different reasons: `Environment` / `AppDomain` / `GC` / `Activator` live directly in the `System` namespace (which is otherwise allowed for primitives), so a namespace-prefix rule cannot reach them without blocking `int` / `string` / `Math`; `Thread` / `ThreadPool` / `Timer` live in `System.Threading` (shared with allowed types like `CancellationToken` and `SemaphoreSlim`), so a prefix on `System.Threading` would block those too. `Environment.Exit` / `FailFast` terminate the host process outright (Core.Scripting-001); `Thread` and `ThreadPool` reintroduce background-fanout vectors that `System.Threading.Tasks` denial closed (Core.Scripting-010 / -012). +`ForbiddenTypeAnalyzer.ForbiddenFullTypeNames` denies type-granularly: `System.Environment`, `System.AppDomain`, `System.GC`, `System.Activator`, `System.Runtime.CompilerServices.Unsafe`, and `System.Runtime.CompilerServices.RuntimeHelpers`. These live in namespaces that are otherwise allowed — `System` (primitives, `Math`) and `System.Runtime.CompilerServices` (compile-time attributes such as `CallerMemberNameAttribute`) — so a namespace-prefix rule cannot reach them. `Environment.Exit` / `FailFast` terminate the host process outright (Core.Scripting-001); `Unsafe.As` bypasses the type system; `RuntimeHelpers.GetUninitializedObject` builds an instance of any type without a constructor. `Thread` / `ThreadPool` / `Timer` / `SemaphoreSlim` and the rest of `System.Threading` are now caught by the namespace prefix above instead of being listed one by one — a script is a synchronous expression with no use for background work or blocking primitives (Core.Scripting-010 / -012, review R05:10). `ForbiddenTypeAnalyzer.ForbiddenMembers` additionally denies `System.Type`'s by-name reflection entry points (`GetType(string)`, `GetMethod(s)`, `InvokeMember`, and the `GetProperty`/`GetField`/`GetMember`/`GetConstructor`/`GetEvent` families), which would otherwise reach a forbidden type without naming it syntactically. #### Known resource limits (accepted trade-offs) -The sandbox cannot prevent a script from **allocating unbounded memory**. A script calling `new byte[int.MaxValue]` repeatedly, or accumulating a large LINQ enumeration, can drive the server process to `OutOfMemoryException` before the 250 ms timeout fires. Script authoring is gated behind the Admin permission as the primary control; the test-harness preview (Stream F.4) allows operators to exercise a script before publishing. Out-of-process script execution is a v3 concern. +The sandbox cannot pre-empt a script. **Memory:** each run's own-thread allocation is sampled (`GC.GetAllocatedBytesForCurrentThread`) and a run over `Scripting:MaxAllocBytesPerRun` (default 64 MB) quarantines the script — but the sample is taken after the run returns, so a script that allocates fast enough (`new byte[int.MaxValue]` in a loop) can still OOM the process first. **CPU:** a tight loop keeps its pool thread after the timeout fires; `ScriptRunGuard` bounds that to one orphan per script by refusing further runs while it spins (Bad `"previous evaluation still running"`) and quarantines after three consecutive timeouts (a refusal is not a strike). Both are mitigations; pre-emptive CPU/memory limits need an out-of-process runner. Script authoring is gated behind the Admin permission as the primary control. See `docs/ScriptEditor.md` §"Runtime safety". -Similarly, **`System.Threading.Tasks` is now denied** (Core.Scripting-003), which prevents `Task.Run` / `Parallel` fan-out that would spawn background work outliving the timeout. However, a tight CPU-bound loop still runs on its thread-pool thread after `WaitAsync` returns — see the `TimedScriptEvaluator` remarks for detail. The orphaned thread is reclaimed when the Roslyn runtime eventually returns; in practice the operator fixes the script once the structured timeout warning appears in `scripts-*.log`. +`System.Threading.Tasks` is denied (Core.Scripting-003), which prevents `Task.Run` / `Parallel` fan-out that would spawn background work outliving the timeout. ### Compile cache (`CompiledScriptCache`) diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs index 3b8d0bb1..96a25da5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs @@ -463,6 +463,10 @@ public static class DraftValidator var (changeTriggered, timerIntervalMs) = ParseTriggers(t.TagConfig); if (timerIntervalMs is > 0) continue; var source = scriptId is not null && scriptSourceById.TryGetValue(scriptId, out var s) ? s : null; + // An unresolved scriptId is already CalculationScriptMissing / CalculationScriptNotFound; with no + // source there are no reads to count, so a change-triggered tag would get a second, misleading + // "no literal ctx.GetTag read" error. (changeTriggered=false + no timer is source-independent.) + if (changeTriggered && source is null) continue; var depCount = EquipmentScriptPaths.ExtractDependencyRefs(source).Count; if (changeTriggered && depCount > 0) continue; errors.Add(new("CalculationNoEffectiveTrigger", diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index ae6352f5..c96c8c87 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -188,6 +188,15 @@ public sealed class ScriptedAlarmEngine : IDisposable /// (DrainDirtyAsync, ShelvingCheckAsync) runs on its token, so a store call that honours /// cancellation returns instead of outliving the engine. private readonly CancellationTokenSource _shutdownCts = new(); + + /// 's token, captured once: background work reads this, never the CTS, + /// so can dispose the CTS without racing a late reader. + private readonly CancellationToken _shutdownToken; + + private volatile bool _shutdownCtsDisposed; + + /// Test hook — whether disposed . + internal bool ShutdownCtsDisposedForTest => _shutdownCtsDisposed; private readonly Dictionary> _alarmsReferencing = new(StringComparer.Ordinal); // tag path -> alarm ids @@ -247,6 +256,7 @@ public sealed class ScriptedAlarmEngine : IDisposable _clock = clock ?? (() => DateTime.UtcNow); _scriptTimeout = scriptTimeout ?? TimedScriptEvaluator.DefaultTimeout; _guard = new ScriptRunGuard(scripting, _engineLogger); + _shutdownToken = _shutdownCts.Token; } /// The run guard every predicate evaluation goes through (diagnostics / tests). @@ -611,7 +621,7 @@ public sealed class ScriptedAlarmEngine : IDisposable // Started outside the lock: DrainDirtyAsync runs synchronously up to its // first real await, and nothing it does should happen under _dirtyLock. - if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(_shutdownCts.Token)); + if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(_shutdownToken)); } /// @@ -754,7 +764,7 @@ public sealed class ScriptedAlarmEngine : IDisposable } finally { _evalGate.Release(); } } - catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + catch (OperationCanceledException) when (_shutdownToken.IsCancellationRequested) { // Dispose cut the drain short after its bound — expected, not a failure. return; @@ -1080,7 +1090,7 @@ public sealed class ScriptedAlarmEngine : IDisposable { if (_disposed) return; var ids = _alarms.Keys.ToArray(); - TrackBackgroundTask(() => ShelvingCheckAsync(ids, _shutdownCts.Token)); + TrackBackgroundTask(() => ShelvingCheckAsync(ids, _shutdownToken)); } /// @@ -1219,7 +1229,7 @@ public sealed class ScriptedAlarmEngine : IDisposable } finally { _evalGate.Release(); } } - catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + catch (OperationCanceledException) when (_shutdownToken.IsCancellationRequested) { return; // Dispose cut the check short after its bound } @@ -1304,6 +1314,16 @@ public sealed class ScriptedAlarmEngine : IDisposable _alarmsReferencing.Clear(); _scratchByAlarmId.Clear(); _lastQualityBucketByAlarmId.Clear(); + + // Release the shutdown CTS once nothing can still be observing it through a registration: every + // drained task has completed. A task still stuck past the grace keeps its registration, so the CTS is + // left to the GC in that case (logged above). + if (toAwait.All(t => t.IsCompleted)) + { + _shutdownCts.Dispose(); + _shutdownCtsDisposed = true; + } + // Dispose every compiled-predicate ALC so the engine's shutdown actually // releases the emitted assemblies. The drain above ensures no evaluator is // mid-call; CompiledScriptCache.Dispose internally guards against use-after- diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs index 17a6f2f6..5cce5fcd 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs @@ -43,6 +43,9 @@ public sealed class ScriptLogRateLimiter _clock = clock ?? (static () => DateTime.UtcNow); } + /// Entries admitted per source per window, as configured (after clamping to at least 1). + public int LimitPerWindow => _limit; + /// The window length. public TimeSpan Window => _window; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs index a7b8f03e..88bdd973 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs @@ -83,7 +83,7 @@ public sealed class ScriptLogTopicSink : ILogEventSink Level: nameof(LogEventLevel.Warning), Message: $"{admission.SuppressedInPreviousWindow} script-log entr{(admission.SuppressedInPreviousWindow == 1 ? "y" : "ies")} " + $"from this source were suppressed in the previous {_limiter.Window.TotalSeconds:F0} s " + - $"(limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); see scripts-*.log for the full stream", + $"(limit {_limiter.LimitPerWindow}/window); see scripts-*.log for the full stream", TimestampUtc: logEvent.Timestamp.UtcDateTime, VirtualTagId: virtualTagId, AlarmId: alarmId, diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs index 905fff13..9f8ef41b 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs @@ -111,7 +111,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba private CalculationTimerScheduler? _timers; private volatile bool _running; private DateTime? _lastComputeUtc; - private bool _disposed; + private volatile bool _disposed; /// Occurs when a calc tag's computed value changes (or transitions Good↔Bad). public event EventHandler? OnDataChange; @@ -538,7 +538,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // _evalGate (never _lock) is held across the run, so the pump and the timer ticks still // evaluate one at a time while no host thread can block behind a script. - await _evalGate.WaitAsync().ConfigureAwait(false); + if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed try { await RunEvaluationAsync(next).ConfigureAwait(false); @@ -566,7 +566,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba PendingEvaluation? prepared; lock (_lock) prepared = PrepareEvaluation(rawPath, DateTime.UtcNow); if (prepared is null) return; - await _evalGate.WaitAsync().ConfigureAwait(false); + if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed try { await RunEvaluationAsync(prepared).ConfigureAwait(false); @@ -577,6 +577,25 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba } } + /// Acquire ; false once has disposed it (a pump + /// iteration or timer tick that raced the teardown simply stops). + private async Task TryEnterEvalGateAsync() + { + if (_disposed) return false; + try + { + await _evalGate.WaitAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + return false; + } + if (!_disposed) return true; + // Dispose landed while we waited (it gave up the gate without disposing it): back out. + _evalGate.Release(); + return false; + } + /// Run one prepared evaluation and publish the outcome. MUST be called under /// and NOT under : the script run is the whole point of the /// split, and the script log + fan-out both invoke subscriber code. @@ -784,11 +803,33 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba StopTimers(); lock (_lock) _pending.Clear(); // Cancel an awaited run first (the guard lets the caller's cancel win over the timeout), then drop - // the compiled scripts. The CTS is not disposed: a pump mid-await may still read its token. + // the compiled scripts. try { _evalCts.Cancel(); } catch (ObjectDisposedException) { } _evaluator.Dispose(); + + // Then take the gate: once Dispose holds it no evaluation is mid-run (the token — read only under + // the gate — has been observed), so the gate and the CTS can be disposed. A late pump iteration or + // timer tick sees _disposed / an ObjectDisposedException in TryEnterEvalGateAsync and stops. Bounded: + // a run that ignores the cancel cannot hold Dispose; in that case both are left to the GC. + if (_evalGate.Wait(DisposeGateTimeout)) + { + _evalCts.Dispose(); + _evalGate.Dispose(); + _evalResourcesDisposed = true; + } + else + { + _logger.LogWarning("Calculation {Driver}: an evaluation did not release the gate within {Timeout} of Dispose; leaving it to the GC", + _driverInstanceId, DisposeGateTimeout); + } } + private static readonly TimeSpan DisposeGateTimeout = TimeSpan.FromSeconds(2); + private volatile bool _evalResourcesDisposed; + + /// Test hook — whether disposed the evaluation gate and its CTS. + internal bool EvalResourcesDisposedForTest => _evalResourcesDisposed; + /// Per-tag published state (guarded by ). private sealed class TagState { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs index 63ea8736..8c7b4f41 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs @@ -377,7 +377,7 @@ public sealed class VirtualTagActor : ReceiveActor { PublishLog("Warning", $"{admission.SuppressedInPreviousWindow} further evaluation failure(s) were suppressed from the script log " + - $"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); last: {reason}"); + $"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {_failureLogLimiter.LimitPerWindow}/window); last: {reason}"); } if (admission.Admitted) PublishLog(level, reason); diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs index b542871f..b68fb1cd 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs @@ -55,7 +55,10 @@ public sealed class DraftValidatorCalculationTests Tags = [CalcTagRow("t-1", "dev-1", "a", "{\"changeTriggered\":true}")], }; - DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationScriptMissing" && e.Context == "t-1"); + var errors = DraftValidator.Validate(draft); + errors.ShouldContain(e => e.Code == "CalculationScriptMissing" && e.Context == "t-1"); + errors.ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger", + "an unresolved script has no reads to count — the missing script is the one error to report"); } [Fact] @@ -70,7 +73,10 @@ public sealed class DraftValidatorCalculationTests Scripts = [], }; - DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationScriptNotFound" && e.Context == "t-1"); + var errors = DraftValidator.Validate(draft); + errors.ShouldContain(e => e.Code == "CalculationScriptNotFound" && e.Context == "t-1"); + errors.ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger", + "an unresolved script has no reads to count — the missing script is the one error to report"); } [Fact] diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs index 7afea07a..2e9b7df6 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs @@ -82,6 +82,7 @@ public sealed class ScriptedAlarmEngineLifecycleTests sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the drain is bounded; a hung store cannot hang shutdown"); await WaitForAsync(() => store.ObservedCancellation, timeoutMs: 2000); store.ObservedCancellation.ShouldBeTrue("the in-flight save was cancelled through the engine's shutdown token"); + eng.ShutdownCtsDisposedForTest.ShouldBeTrue("the cancelled save completed inside the grace, so the CTS is released"); } [Fact] @@ -105,6 +106,8 @@ public sealed class ScriptedAlarmEngineLifecycleTests store.SaveInProgress.ShouldBeFalse(); store.ObservedCancellation.ShouldBeFalse("a save that completes within the bound is never cancelled"); store.Saved.ShouldContain("A"); + eng.ShutdownCtsDisposedForTest.ShouldBeTrue("every background task completed, so the shutdown CTS is disposed"); + Should.NotThrow(() => eng.OnUpstreamChange("Temp", new DataValueSnapshot(1, 0u, DateTime.UtcNow, DateTime.UtcNow))); } private static async Task WaitForAsync(Func cond, int timeoutMs = 5000) diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs index 3f560e40..70674942 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs @@ -59,6 +59,9 @@ public sealed class ScriptLogRateLimiterTests ScriptLogRateLimiter.DefaultLimitPerWindow.ShouldBe(20); ScriptLogRateLimiter.DefaultWindow.ShouldBe(TimeSpan.FromMinutes(1)); var limiter = new ScriptLogRateLimiter(); + limiter.LimitPerWindow.ShouldBe(20); + new ScriptLogRateLimiter(limitPerWindow: 3).LimitPerWindow.ShouldBe(3); + new ScriptLogRateLimiter(limitPerWindow: 0).LimitPerWindow.ShouldBe(1, "values below 1 are treated as 1"); for (var i = 0; i < 20; i++) limiter.Admit("s").Admitted.ShouldBeTrue(); limiter.Admit("s").Admitted.ShouldBeFalse(); } @@ -101,6 +104,7 @@ public sealed class ScriptLogRateLimiterTests var summary = s1[2]; summary.Message.ShouldContain("3"); summary.Message.ShouldContain("suppressed"); + summary.Message.ShouldContain("limit 2/window", Case.Sensitive, "the summary names the CONFIGURED limit, not the default"); summary.VirtualTagId.ShouldBe("V1", "the summary carries the source's identity so the page attributes it"); summary.Level.ShouldBe("Warning"); s1[3].Message.ShouldBe("chatty again"); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs index 88015ffc..0372ea15 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs @@ -50,6 +50,29 @@ public sealed class CalculationDriverRemediationTests driver.IsEvaluationIdle.ShouldBeTrue("the calc driver's evaluation pump should have drained"); } + /// Fix round 1 minor: Dispose releases the evaluation gate and its cancellation source — even + /// with a runaway run in flight (the guard lets the caller's cancel win, so the awaited run returns + /// promptly) — and a trigger arriving after Dispose is harmless. + [Fact] + public async Task Dispose_with_a_run_in_flight_disposes_the_eval_gate_and_a_late_trigger_is_harmless() + { + var (driver, _, _, _) = Build( + CalcTag("calc/e/spin", "var v = ctx.GetTag(\"src/a\").Value; var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 1;")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); + var deadline = DateTime.UtcNow.AddSeconds(10); + while (driver.IsEvaluationIdle && DateTime.UtcNow < deadline) await Task.Delay(5); + driver.IsEvaluationIdle.ShouldBeFalse("positive control: a run is in flight when Dispose lands"); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + driver.Dispose(); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the caller's cancel releases the awaited run; Dispose is not held for the 6 s spin"); + driver.EvalResourcesDisposedForTest.ShouldBeTrue(); + + Should.NotThrow(() => driver.OnDependencyValue("src/a", 2, StatusGood, DateTime.UtcNow)); + Should.NotThrow(() => driver.Dispose()); + } + [Fact] public async Task A_compile_failure_is_remembered_and_logged_once_across_repeated_triggers() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs index 74ee2e32..c1a6253e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs @@ -43,6 +43,7 @@ public sealed class VirtualTagActorFailureLogRateLimitTests : RuntimeActorTestBa var entries = capture.ToArray(); entries[3].Message.ShouldContain("7"); entries[3].Message.ShouldContain("suppressed"); + entries[3].Message.ShouldContain("limit 3/window", Case.Sensitive, "the summary names the CONFIGURED limit, not the default"); entries[3].VirtualTagId.ShouldBe("vt-chatty"); entries[4].Message.ShouldBe("boom"); } From f7e7ba0263045b1ce0ae1bbba3a60b3ead7bc12a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:42:29 -0400 Subject: [PATCH 21/27] fix(controlplane): audit writer drops only already-persisted EventIds, never the whole SQL Server batch SQL Server's DbUpdateException for one duplicate names every entry in the statement batch, so detaching ex.Entries dropped the whole flush. On a row-level rejection the writer now asks the database which of the batch's EventIds it already holds, drops exactly those, re-saves the rest, and falls back to row-by-row only if a non-duplicate row is still refused. Fresh context + entities per attempt. Pinned against the real central SQL Server (new SqlServerTestDatabase harness) and a faithful whole-batch interceptor. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Audit/AuditWriterActor.cs | 134 +++++++++++------- .../AuditWriterActorRejectedRowTests.cs | 109 ++++++++++++-- .../Harness/SqlServerTestDatabase.cs | 77 ++++++++++ 3 files changed, 255 insertions(+), 65 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs index 0e20da2f..3f0aa914 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs @@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; /// Dedup is two-layer: in-buffer (the below collapses /// duplicate EventIds before flush) and at the database via the filtered unique index /// UX_ConfigAuditLog_EventId (cross-restart safety — a retry of an already-flushed -/// batch hits the constraint; detaches the rejected row -/// and re-saves, so the duplicate is dropped without losing the rest of the batch). +/// batch hits the constraint; drops exactly the already-persisted EventIds +/// and re-saves the rest, so the duplicate is dropped without losing the rest of the batch). /// /// Implements the shared seam: is a /// best-effort, never-throwing entry point that simply Tells this actor and returns @@ -92,22 +92,7 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter try { - using var db = _dbFactory.CreateDbContext(); - foreach (var evt in snapshot) - { - db.ConfigAuditLogs.Add(new ConfigAuditLog - { - Timestamp = evt.OccurredAtUtc.UtcDateTime, - Principal = evt.Actor, - EventType = $"{evt.Category}:{evt.Action}", - NodeId = evt.SourceNode, - DetailsJson = evt.DetailsJson, - EventId = evt.EventId, - CorrelationId = evt.CorrelationId, - Outcome = evt.Outcome.ToString(), - }); - } - var dropped = SaveDetachingRejectedRows(db); + var dropped = SaveBatch(snapshot); _log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped); } catch (Exception ex) @@ -116,46 +101,93 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter } } + private static ConfigAuditLog ToRow(AuditEvent evt) => new() + { + Timestamp = evt.OccurredAtUtc.UtcDateTime, + Principal = evt.Actor, + EventType = $"{evt.Category}:{evt.Action}", + NodeId = evt.SourceNode, + DetailsJson = evt.DetailsJson, + EventId = evt.EventId, + CorrelationId = evt.CorrelationId, + Outcome = evt.Outcome.ToString(), + }; + /// - /// Saves the batch, detaching and retrying past any row the database rejects (R03:2.6). One - /// statement batch used to carry the whole flush: a single duplicate EventId (any at-least-once - /// producer — the audit seam is Tell-based through a singleton proxy that re-delivers across a - /// hand-over) threw for all 500 rows and the class remarks' per-row dedup claim was not what ran. + /// Persists one flush, dropping ONLY the rows the database will not take (R03:2.6, fix round 1 / I4). /// - /// The context holding the batch. + /// The flush. /// How many rows were dropped. /// - /// Only a that NAMES its entries is handled here — that is the shape a - /// constraint violation takes (SQL 2601/2627 for the unique index). A failure with no entries (the - /// connection died, a deadlock) is the whole-batch failure it always was and propagates to the - /// caller's catch. Bounded by the batch size so a store that rejects every row still terminates. + /// + /// Why not detach ex.Entries. SQL Server inserts the batch as one statement batch, so + /// the raised for ONE duplicate EventId names EVERY entry in + /// the batch. Detaching what it names dropped the whole flush — the exact loss the fix was for + /// (pinned against a real SQL Server by On_sql_server_a_duplicate_event_id_costs_only_that_row). + /// + /// + /// So, on a rejection that names entries (a constraint violation; 2601/2627 for the unique index): + /// (1) ask the database which of the batch's EventIds it already holds and drop exactly those — + /// the at-least-once re-delivery the dedup contract expects; (2) re-save the rest as a batch; (3) if + /// that is still rejected (some OTHER per-row constraint), fall back to row-by-row and drop only the + /// rows that fail on their own. Every attempt uses a fresh context and fresh entities, so no state + /// from a failed save leaks into the next one. A failure that names no entry (connection loss, + /// deadlock) propagates — the whole-batch failure it always was. Bounded: at most 2 + N saves. + /// /// - private int SaveDetachingRejectedRows(OtOpcUaConfigDbContext db) + private int SaveBatch(IReadOnlyList batch) { - var dropped = 0; - var attempts = 0; - var maxAttempts = db.ChangeTracker.Entries().Count() + 1; - while (true) + if (TrySave(batch, out var first)) return 0; + _log.Debug(first, "AuditWriter: a batch of {Count} was rejected; dropping only already-persisted EventIds", batch.Count); + + HashSet existing; + using (var db = _dbFactory.CreateDbContext()) { - try - { - db.SaveChanges(); - return dropped; - } - catch (DbUpdateException ex) when (ex.Entries.Count > 0 && ++attempts < maxAttempts) - { - var duplicate = IsUniqueViolation(ex); - foreach (var entry in ex.Entries) - { - var eventId = (entry.Entity as ConfigAuditLog)?.EventId; - if (duplicate) - _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", eventId); - else - _log.Warning(ex, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", eventId); - entry.State = EntityState.Detached; - dropped++; - } - } + var ids = batch.Select(e => (Guid?)e.EventId).ToList(); + existing = db.ConfigAuditLogs + .AsNoTracking() + .Where(r => ids.Contains(r.EventId)) + .Select(r => r.EventId!.Value) + .ToHashSet(); + } + foreach (var id in existing) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", id); + + var remaining = batch.Where(e => !existing.Contains(e.EventId)).ToList(); + var dropped = batch.Count - remaining.Count; + if (remaining.Count == 0 || TrySave(remaining, out _)) return dropped; + + // Still rejected with no duplicate left to blame: isolate the offending row(s). + foreach (var evt in remaining) + { + if (TrySave([evt], out var rowError)) continue; + dropped++; + if (IsUniqueViolation(rowError!)) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", evt.EventId); + else + _log.Warning(rowError, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", evt.EventId); + } + return dropped; + } + + /// Saves on a fresh context; a rejection that names entries is reported, anything else throws. + /// The rows to save. + /// The row-level rejection, when this returns false. + /// when saved. + private bool TrySave(IReadOnlyList events, out DbUpdateException? rejection) + { + using var db = _dbFactory.CreateDbContext(); + foreach (var evt in events) db.ConfigAuditLogs.Add(ToRow(evt)); + try + { + db.SaveChanges(); + rejection = null; + return true; + } + catch (DbUpdateException ex) when (ex.Entries.Count > 0 || IsUniqueViolation(ex)) + { + rejection = ex; + return false; } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs index bd64161a..aba37648 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs @@ -26,7 +26,7 @@ public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase var dbName = Guid.NewGuid().ToString("N"); var root = new InMemoryDatabaseRoot(); var duplicate = Guid.NewGuid(); - var rejecting = new RejectingDbFactory(dbName, root, new RejectOnce(duplicate)); + var rejecting = new RejectingDbFactory(dbName, root, new AlwaysRejectBatchContaining(duplicate)); var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); @@ -46,6 +46,68 @@ public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase persisted.ShouldContain(ids[8]); } + /// + /// Fix round 1 / I4 — a FAITHFUL SQL Server shape: the provider sends the batch as one statement batch, + /// so the for one duplicate names EVERY entry in it. Detaching + /// ex.Entries therefore dropped the whole flush. The writer must drop only the rows whose + /// EventId is already persisted and keep the rest. + /// + [Fact] + public void A_whole_batch_rejection_drops_only_the_rows_already_persisted() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var factory = new RejectingDbFactory(dbName, root, new RejectWholeBatchOnDuplicate()); + var duplicate = Guid.NewGuid(); + using (var seed = factory.CreateDbContext()) + { + seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate }); + seed.SaveChanges(); + } + var actor = Sys.ActorOf(AuditWriterActor.Props(factory)); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor); + + using var db = factory.CreateDbContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(21, "the 20 new rows land beside the one that was already there"); + persisted.Count(id => id == duplicate).ShouldBe(1); + db.ConfigAuditLogs.Single(r => r.EventId == duplicate).Principal.ShouldBe("earlier"); + } + + /// The same contract against the REAL provider and the real filtered unique index. + [Fact] + [Trait("Category", "SchemaCompliance")] + public void On_sql_server_a_duplicate_event_id_costs_only_that_row() + { + using var sql = new SqlServerTestDatabase(); + var duplicate = Guid.NewGuid(); + using (var seed = sql.NewContext()) + { + seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate }); + seed.SaveChanges(); + } + var actor = Sys.ActorOf(AuditWriterActor.Props(sql.Factory())); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor, PresenceBudget); + + using var db = sql.NewContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(21); + persisted.Count(id => id == duplicate).ShouldBe(1); + foreach (var id in ids) persisted.ShouldContain(id); + } + /// A failure that names no entry is a whole-batch failure, exactly as before. [Fact] public void A_connection_level_failure_still_drops_the_batch_without_looping() @@ -71,24 +133,43 @@ public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(), }; - /// Throws a naming the entry for one EventId, the first time it is saved. - private sealed class RejectOnce(Guid eventId) : ISaveChangesInterceptor + /// + /// A row the database refuses on its own terms (not a duplicate — e.g. a per-row constraint), every + /// time it is in the save. Faithful to SQL Server's batch shape: the exception names EVERY added + /// entry, so only a row-by-row retry can isolate the offender. + /// + private sealed class AlwaysRejectBatchContaining(Guid eventId) : ISaveChangesInterceptor { - private int _fired; - public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) { - if (_fired == 1 || eventData.Context is null) return result; - var entry = eventData.Context.ChangeTracker.Entries() - .FirstOrDefault(e => e.Entity.EventId == eventId); - if (entry is null) return result; - _fired = 1; - // GetInfrastructure() hands back the pubternal InternalEntityEntry, which is the IUpdateEntry the - // real provider puts on the exception; the test only needs that one cast. + if (eventData.Context is null) return result; + var added = eventData.Context.ChangeTracker.Entries() + .Where(e => e.State == EntityState.Added).ToList(); + if (!added.Any(e => e.Entity.EventId == eventId)) return result; #pragma warning disable EF1001 - var updateEntry = (IUpdateEntry)entry.GetInfrastructure(); + var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList(); #pragma warning restore EF1001 - throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601)", new[] { updateEntry }); + throw new DbUpdateException("The statement conflicted with a CHECK constraint (simulated, whole batch)", entries); + } + } + + /// + /// Models SQL Server's batched insert: when ANY added row's EventId is already stored, the whole save + /// fails with an exception naming EVERY added entry (the InMemory provider enforces no unique index, + /// so this interceptor is the constraint). + /// + private sealed class RejectWholeBatchOnDuplicate : ISaveChangesInterceptor + { + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) + { + if (eventData.Context is not OtOpcUaConfigDbContext ctx) return result; + var added = ctx.ChangeTracker.Entries().Where(e => e.State == EntityState.Added).ToList(); + var ids = added.Select(e => e.Entity.EventId).Where(id => id != null).ToList(); + if (ids.Count == 0 || !ctx.ConfigAuditLogs.AsNoTracking().Any(r => ids.Contains(r.EventId))) return result; +#pragma warning disable EF1001 + var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList(); +#pragma warning restore EF1001 + throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601, whole batch)", entries); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs new file mode 100644 index 00000000..3415bb99 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs @@ -0,0 +1,77 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +/// +/// A migrated, throw-away config database on the central SQL Server, for the few ControlPlane behaviours +/// the InMemory provider cannot model (a SQL Server DbUpdateException names the WHOLE statement +/// batch; collation-ordered reads). Same server / credentials / env overrides as +/// Configuration.Tests.SchemaComplianceFixture (OTOPCUA_CONFIG_TEST_SERVER, +/// OTOPCUA_CONFIG_TEST_SA_PASSWORD; default the shared docker host's dev SA), and the same database +/// name shape, so that suite's 24 h orphan reaper also reclaims a database a killed run leaks here. +/// +public sealed class SqlServerTestDatabase : IDisposable +{ + private const string DefaultServer = "10.100.0.35,14330"; + private const string DefaultSaPassword = "OtOpcUaDev_2026!"; + + /// Gets the connection string of the migrated database. + public string ConnectionString { get; } + + private readonly string _databaseName; + + /// Creates and migrates the database. + public SqlServerTestDatabase() + { + var server = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SERVER") ?? DefaultServer; + var saPassword = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SA_PASSWORD") ?? DefaultSaPassword; + _databaseName = $"OtOpcUaConfig_Test_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}"; + ConnectionString = + $"Server={server};Database={_databaseName};User Id=sa;Password={saPassword};TrustServerCertificate=True;Encrypt=False;"; + using var ctx = NewContext(); + ctx.Database.Migrate(); + } + + /// A new context on the database. + /// The context. + public OtOpcUaConfigDbContext NewContext() => + new(new DbContextOptionsBuilder().UseSqlServer(ConnectionString).Options); + + /// A factory over . + /// The factory. + public IDbContextFactory Factory() => new ContextFactory(this); + + /// Drops the database (best effort; a leak is reaped by the Configuration suite). + public void Dispose() + { + try + { + SqlConnection.ClearAllPools(); + var master = new SqlConnectionStringBuilder(ConnectionString) { InitialCatalog = "master" }.ConnectionString; + using var conn = new SqlConnection(master); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = @" +DECLARE @sql nvarchar(max); +IF DB_ID(@name) IS NOT NULL +BEGIN + SET @sql = N'ALTER DATABASE ' + QUOTENAME(@name) + N' SET SINGLE_USER WITH ROLLBACK IMMEDIATE; ' + + N'DROP DATABASE ' + QUOTENAME(@name) + N';'; + EXEC sp_executesql @sql; +END"; + cmd.Parameters.Add(new SqlParameter("@name", _databaseName)); + cmd.ExecuteNonQuery(); + } + catch (Exception) + { + // Best effort — see the class remarks. + } + } + + private sealed class ContextFactory(SqlServerTestDatabase owner) : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => owner.NewContext(); + } +} From 9466b8b242f80d36f3ef6c481a6058e9eee46722 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:46:54 -0400 Subject: [PATCH 22/27] fix(controlplane): close a lost dispatch from the in-flight check; guard terminal rows; blob-free status writes I7: a Dispatching row committed after the coordinator's PreStart whose Tell never arrived had no owner and refused every later deploy. The in-flight check (now fully off the admin singleton's thread, serialised by the actor-local compose flag alone) closes a Dispatching row with zero NodeDeploymentState rows that is past the apply deadline as TimedOut (optimistic on RowVersion) and proceeds. HandleDispatch ignores a dispatch for a Sealed/PartiallyFailed/TimedOut row, so a late or duplicated dispatch cannot flip it back. Coordinator status/reason writes attach a key+RowVersion stub instead of loading the multi-MB ArtifactBlob; proven on the real SQL Server (blob untouched, seal lands). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdminOperations/AdminOperationsActor.cs | 172 +++++++++++++----- .../Coordinators/ConfigPublishCoordinator.cs | 84 +++++++-- .../ConfigPublishCoordinatorRecoveryTests.cs | 135 ++++++++++++++ 3 files changed, 332 insertions(+), 59 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index e78c926e..81f09dae 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -11,6 +11,7 @@ using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Configuration.Validation; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Scripting; @@ -28,6 +29,12 @@ public sealed class AdminOperationsActor : ReceiveActor private readonly IReadOnlyDictionary _probesByType; private readonly TagConfigValidationMode _tagConfigValidationMode; + /// + /// The coordinator's apply deadline. A Dispatching row with no expected-ack rows that is older + /// than this is a lost dispatch, closed by the in-flight check (fix round 1 / I7). + /// + private readonly TimeSpan _applyDeadline; + /// /// Where central→node commands are handed off (per-cluster mesh), or /// to publish on the mediator directly. @@ -48,15 +55,17 @@ public sealed class AdminOperationsActor : ReceiveActor /// Central comm actor commands are handed to, or to publish on the /// DistributedPubSub mediator directly (the behaviour before the per-cluster mesh). /// + /// The coordinator's apply deadline; defaults to . /// Props configured to create an AdminOperationsActor. public static Props Props( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, - IActorRef? meshRouter = null) => + IActorRef? meshRouter = null, + TimeSpan? applyDeadline = null) => Akka.Actor.Props.Create(() => - new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter)); + new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter, applyDeadline)); /// Initializes a new instance of the AdminOperationsActor. /// Factory for creating config database contexts. @@ -64,13 +73,16 @@ public sealed class AdminOperationsActor : ReceiveActor /// Driver probes registered in DI; keyed by DriverType (case-insensitive). /// Deploy-gate TagConfig strictness mode (Warn default | Error opt-in). /// Central comm actor, or for the mediator. + /// The coordinator's apply deadline; defaults to . public AdminOperationsActor( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, - IActorRef? meshRouter = null) + IActorRef? meshRouter = null, + TimeSpan? applyDeadline = null) { + _applyDeadline = applyDeadline ?? ConfigPublishCoordinator.DefaultApplyDeadline; _dbFactory = dbFactory; _coordinator = coordinator; _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase); @@ -201,16 +213,17 @@ public sealed class AdminOperationsActor : ReceiveActor } /// - /// Serialised part of StartDeployment: the in-flight check. Everything after it — the - /// snapshot read, validation, gzip and the row write — runs on the thread pool and comes back as a - /// / message, so a large compose no - /// longer suspends the mailbox (R03:3.1) and alarm acks keep flowing while it runs. + /// StartDeployment. The only serialised part is the actor-local + /// flag; the DB in-flight check, the snapshot read, validation, gzip and the row write all run on the + /// thread pool and come back as a / + /// message, so a slow or blocked ConfigDb never suspends the mailbox (R03:3.1; fix round 1 moved the + /// in-flight query off the actor thread too) and alarm acks keep flowing while it runs. /// /// - /// is what keeps the check correct once the compose is off-actor: the - /// Deployment row is written at the END of the compose, so a second request arriving mid-compose - /// would pass a DB-only check and two artifacts would race for one coordinator. The flag is cleared - /// when the outcome message lands, on both branches. + /// is what keeps the check correct once it is off-actor: it is set + /// BEFORE the off-actor work starts and cleared when the outcome message lands (both branches), so a + /// second request is refused in memory for the whole check-plus-compose and two artifacts can never + /// race for one coordinator. /// private void HandleStartDeployment(StartDeployment msg) { @@ -226,49 +239,118 @@ public sealed class AdminOperationsActor : ReceiveActor return; } - try - { - // Refuse if any deployment is already in flight — keeps the coordinator's state - // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. One indexed - // projection (IX_Deployment_Status), synchronous on purpose: it IS the serialised part. - using var db = _dbFactory.CreateDbContext(); - var inflight = db.Deployments - .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) - .Select(d => d.DeploymentId) - .FirstOrDefault(); - if (inflight != Guid.Empty) - { - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.AnotherDeploymentInFlight, - DeploymentId: new DeploymentId(inflight), - RevisionHash: null, - Message: $"Deployment {inflight:N} is still in flight.", - msg.CorrelationId)); - return; - } - } - catch (Exception ex) - { - _log.Error(ex, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.CreatedBy, msg.CorrelationId); - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Rejected, - DeploymentId: null, - RevisionHash: null, - Message: ServerFault("StartDeployment", msg.CorrelationId), - msg.CorrelationId)); - return; - } - _composeInFlight = msg; // Captured here: Context/Sender are not available on the thread-pool thread. var sourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown"; var self = Self; - Task.Run(() => ComposeDeploymentAsync(msg, sourceNode)) + Task.Run(async () => + { + var refusal = await CheckInFlightAsync(msg).ConfigureAwait(false); + return refusal ?? await ComposeDeploymentAsync(msg, sourceNode).ConfigureAwait(false); + }) .PipeTo(self, success: composed => new DeployComposed(msg, replyTo, composed.Result, composed.Dispatch), failure: ex => new DeployComposeFailed(msg, replyTo, ex)); } + /// + /// The DB half of the in-flight check, off the actor thread. Refuses while a deployment is in flight — + /// keeping the coordinator's state unambiguous — EXCEPT a lost dispatch, which it closes first. + /// + /// The request. + /// The refusal, or null to proceed. + /// + /// + /// Lost dispatch (fix round 1 / I7). A Dispatching row with ZERO + /// NodeDeploymentState rows was never taken by the coordinator (its HandleDispatch + /// seeds the expected set and flips the status in one SaveChanges). The coordinator's PreStart + /// recovers such a row — but only one that already existed when it started. A row committed AFTER + /// PreStart whose Tell never arrived (this actor died between the compose's commit and + /// ) has no owner at all, and refused every later deploy forever. + /// Once it is past the apply deadline nothing can still be legitimately dispatching it, so it is + /// closed TimedOut with a reason and the new deploy proceeds. A late DispatchDeployment + /// for it is ignored by the coordinator's terminal-state guard. + /// + /// + /// One indexed projection (IX_Deployment_Status) plus, only for a Dispatching row past the + /// deadline, a count and a tracked update. + /// + /// + private async Task CheckInFlightAsync(StartDeployment msg) + { + await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false); + var inflight = await db.Deployments + .AsNoTracking() + .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) + .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, d.Status, d.CreatedAtUtc }) + .ToListAsync() + .ConfigureAwait(false); + + foreach (var row in inflight) + { + if (row.Status == DeploymentStatus.Dispatching + && DateTime.UtcNow - row.CreatedAtUtc > _applyDeadline + && !await db.NodeDeploymentStates.AnyAsync(s => s.DeploymentId == row.DeploymentId).ConfigureAwait(false) + && await CloseLostDispatchAsync(db, row.DeploymentId).ConfigureAwait(false)) + { + continue; + } + + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.AnotherDeploymentInFlight, + DeploymentId: new DeploymentId(row.DeploymentId), + RevisionHash: null, + Message: $"Deployment {row.DeploymentId:N} is still in flight.", + msg.CorrelationId), + Dispatch: null); + } + return null; + } + + /// Closes a lost dispatch as TimedOut, unless it moved on meanwhile (optimistic, on its RowVersion). + /// The context. + /// The row. + /// when the row is now closed. + private async Task CloseLostDispatchAsync(OtOpcUaConfigDbContext db, Guid deploymentId) + { + var stamp = await db.Deployments + .AsNoTracking() + .Where(d => d.DeploymentId == deploymentId && d.Status == DeploymentStatus.Dispatching) + .Select(d => new { d.RowVersion }) + .FirstOrDefaultAsync() + .ConfigureAwait(false); + if (stamp is null) return true; // no longer Dispatching — not in flight as far as this check goes + + // Stub update: the row's ArtifactBlob is never read just to change its status. + var stub = new Deployment + { + DeploymentId = deploymentId, RevisionHash = string.Empty, CreatedBy = string.Empty, + RowVersion = stamp.RowVersion, + Status = DeploymentStatus.TimedOut, + FailureReason = + "Dispatch was lost before any node was expected to ack (the coordinator was never told, or never " + + "took it) and the apply deadline has passed; closed by the next StartDeployment. Redeploy.", + }; + var entry = db.Deployments.Attach(stub); + entry.Property(d => d.Status).IsModified = true; + entry.Property(d => d.FailureReason).IsModified = true; + try + { + await db.SaveChangesAsync().ConfigureAwait(false); + } + catch (DbUpdateConcurrencyException) + { + // The coordinator took it after all (status flipped, RowVersion moved): it is genuinely in flight. + return false; + } + _log.Warning( + "StartDeployment: closed lost dispatch {DeploymentId} as TimedOut — Dispatching with no expected-ack rows " + + "past the {Deadline} apply deadline (the coordinator never took it)", deploymentId, _applyDeadline); + return true; + } + private void HandleDeployComposed(DeployComposed msg) { _composeInFlight = null; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index 3c99e998..49b54df4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -234,6 +234,26 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers private void HandleDispatch(DispatchDeployment msg) { + // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already + // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message + // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it. + using (var check = _dbFactory.CreateDbContext()) + { + var status = check.Deployments + .AsNoTracking() + .Where(d => d.DeploymentId == msg.DeploymentId.Value) + .Select(d => (DeploymentStatus?)d.Status) + .FirstOrDefault(); + if (status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) + { + _log.Warning( + "Ignoring DispatchDeployment {Id}: the deployment is already {Status} (a late or duplicated dispatch)", + msg.DeploymentId, status); + if (_current == msg.DeploymentId) ResetForNext(); + return; + } + } + _current = msg.DeploymentId; _acks.Clear(); _degraded.Clear(); @@ -371,11 +391,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // same as clean, though: the deployment's reason names the degraded nodes (per-node detail is on // each NodeDeploymentState row) and the seal logs at Warning. var nodes = _degraded.Select(n => n.Value).Order(StringComparer.OrdinalIgnoreCase).ToList(); - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); - if (d is not null) - d.FailureReason = Truncate( - $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " + - "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it."); + SetFailureReason(db, Truncate( + $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " + + "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it.")); db.SaveChanges(); _log.Warning( "Deployment {Id} sealed DEGRADED (acks={Count}) — the address-space apply was degraded on: {Nodes}", @@ -420,11 +438,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut); if (_expectedAcks.Count == 0) { - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); - if (d is not null) - d.FailureReason = Truncate( - "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + - "hand-over mid-dispatch) and the apply deadline has passed. Redeploy."); + SetFailureReason(db, Truncate( + "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + + "hand-over mid-dispatch) and the apply deadline has passed. Redeploy.")); db.SaveChanges(); _log.Warning( "Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " + @@ -459,13 +475,53 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers public sealed record DeadlineElapsed(DeploymentId DeploymentId); + /// + /// Stages a status change WITHOUT materialising the row: the row carries the whole ArtifactBlob + /// (varbinary(max), multi-MB on a real fleet), which no status transition reads (fix round 1). A stub + /// carrying only the key and the row's current RowVersion is attached and the changed columns marked + /// modified, so the UPDATE writes those columns and keeps the optimistic-concurrency check. The staged + /// stub stays tracked so can add to the same UPDATE. + /// private static void UpdateDeploymentStatus( OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false) { - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == id.Value); - if (d is null) return; - d.Status = status; - if (sealNow) d.SealedAtUtc = DateTime.UtcNow; + var entry = TrackStub(db, id); + if (entry is null) return; + entry.Entity.Status = status; + entry.Property(x => x.Status).IsModified = true; + if (sealNow) + { + entry.Entity.SealedAtUtc = DateTime.UtcNow; + entry.Property(x => x.SealedAtUtc).IsModified = true; + } + } + + /// Stages the current deployment's failure reason onto the stub staged. + private void SetFailureReason(OtOpcUaConfigDbContext db, string reason) + { + if (_current is null) return; + var entry = TrackStub(db, _current.Value); + if (entry is null) return; + entry.Entity.FailureReason = reason; + entry.Property(x => x.FailureReason).IsModified = true; + } + + /// The tracked blob-free stub for a deployment, attaching one on first use; null when the row is gone. + private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry? TrackStub( + OtOpcUaConfigDbContext db, DeploymentId id) + { + var tracked = db.ChangeTracker.Entries().FirstOrDefault(e => e.Entity.DeploymentId == id.Value); + if (tracked is not null) return tracked; + var stamp = db.Deployments + .AsNoTracking() + .Where(x => x.DeploymentId == id.Value) + .Select(x => new { x.RowVersion }) + .FirstOrDefault(); + if (stamp is null) return null; + return db.Deployments.Attach(new Deployment + { + DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = stamp.RowVersion, + }); } /// diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs index 86f6efa7..0added3d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -176,6 +176,141 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes }, duration: PresenceBudget); } + /// + /// Fix round 1 / I7 — the window PreStart cannot see: the Dispatching row is committed AFTER the + /// coordinator's PreStart ran, and the coordinator is never told (the admin singleton died between + /// its commit and the Tell). Nothing would ever close it. The in-flight check now closes a + /// Dispatching row with NO expected-ack rows once it is past the apply deadline, so the next + /// StartDeployment is Accepted (and seals — positive control on the pipeline). + /// + [Fact] + public async Task Lost_dispatch_committed_after_PreStart_is_closed_by_the_next_StartDeployment() + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, "site-a-1:4053"); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart saw nothing + + var stale = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + dbFactory, coordinator, Enumerable.Empty(), applyDeadline: NoTimeout)); + + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var result = ExpectMsg(PresenceBudget); + result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, result.Message); + + using (var db = dbFactory.CreateDbContext()) + { + var row = db.Deployments.Single(d => d.DeploymentId == stale.Value); + row.Status.ShouldBe(DeploymentStatus.TimedOut); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("lost"); + } + + coordinator.Tell(new ApplyAck(result.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == result.DeploymentId.Value.Value).Status + .ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// Control for the closure: the same row INSIDE the deadline is still in flight and still refuses. + [Fact] + public void Dispatching_row_inside_the_deadline_still_refuses_a_new_deployment() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe(); + var pending = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(-5)); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + dbFactory, coordinator.Ref, Enumerable.Empty(), applyDeadline: NoTimeout)); + + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var result = ExpectMsg(PresenceBudget); + result.Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight); + result.DeploymentId.ShouldBe(pending); + + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching); + } + + /// + /// Fix round 1 minor — a late or duplicated DispatchDeployment for a row that is already + /// terminal (it raced the lost-dispatch closure above, or a re-delivery after a seal) must not flip + /// it back to AwaitingApplyAcks or seed an expected set. Positive control: a live row dispatched right + /// after it IS seeded (so the first message was processed, not merely not-yet-processed). + /// + [Theory] + [InlineData(DeploymentStatus.Sealed)] + [InlineData(DeploymentStatus.TimedOut)] + [InlineData(DeploymentStatus.PartiallyFailed)] + public void Dispatch_of_a_terminal_row_is_ignored(DeploymentStatus terminal) + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, "site-a-1:4053"); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + var done = SeedDeployment(dbFactory, terminal, DateTime.UtcNow.AddMinutes(-1)); + coordinator.Tell(new DispatchDeployment(done, TestRevision, CorrelationId.NewId())); + var live = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == live.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + using var verify = dbFactory.CreateDbContext(); + verify.Deployments.Single(d => d.DeploymentId == done.Value).Status.ShouldBe(terminal); + verify.NodeDeploymentStates.Count(s => s.DeploymentId == done.Value).ShouldBe(0); + } + + /// + /// Fix round 1 minor — status transitions update a blob-free stub (key + RowVersion) instead of + /// materialising the multi-MB row. Proven on the REAL provider, where the stub's RowVersion is the + /// optimistic-concurrency predicate: dispatch → ack → Sealed lands, and the artifact bytes are + /// untouched by the partial UPDATEs. + /// + [Fact] + [Trait("Category", "SchemaCompliance")] + public void On_sql_server_status_transitions_do_not_touch_the_artifact_blob() + { + using var sql = new SqlServerTestDatabase(); + var factory = sql.Factory(); + SeedNodes(factory, "site-a-1:4053"); + var blob = Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray(); + var id = DeploymentId.NewId(); + using (var db = sql.NewContext()) + { + db.Deployments.Add(new Deployment + { + DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = DeploymentStatus.Dispatching, + CreatedBy = "test", ArtifactBlob = blob, + }); + db.SaveChanges(); + } + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(factory, NoTimeout)); + AwaitAssert(() => + { + using var db = sql.NewContext(); + db.Deployments.AsNoTracking().Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); // PreStart re-drives the un-dispatched row + coordinator.Tell(new ApplyAck(id, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = sql.NewContext(); + var row = db.Deployments.AsNoTracking().Single(); + row.Status.ShouldBe(DeploymentStatus.Sealed); + row.SealedAtUtc.ShouldNotBeNull(); + row.ArtifactBlob.ShouldBe(blob); + }, duration: PresenceBudget); + } + private static DeploymentId SeedDeployment( IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) { From 2adeacc65582d4704cbc254c4c6e4194058c7d4a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:48:40 -0400 Subject: [PATCH 23/27] fix(controlplane): keep the revision hash byte-identical to the pre-refactor composer The one-snapshot refactor re-sorted every collection in memory with an ordinal comparer, while the old composer's SQL ORDER BY used the database's case-insensitive collation; mixed-case keys therefore re-hashed an unchanged configuration (spurious drift fleet-wide, a "new" revision on a no-op redeploy). The snapshot factory now reads every composed table ORDER BY its natural key server-side and Compose serialises in the order given. Pinned byte-for-byte against a verbatim replica of the old composer on the real SQL Server. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Validation/DraftSnapshotFactory.cs | 40 +++++---- .../AdminOperations/ConfigComposer.cs | 51 ++++++----- ...nfigComposerSqlServerHashStabilityTests.cs | 87 +++++++++++++++++++ 3 files changed, 139 insertions(+), 39 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs index 154d664c..30873aca 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs @@ -56,6 +56,12 @@ public static class DraftSnapshotFactory } } + // Every composed table is read ORDERED BY ITS NATURAL KEY, server-side. That order IS the deployment + // artifact's order (ConfigComposer.Compose serialises the snapshot as given): it is exactly the order the + // pre-R03:3.2 composer's own queries produced — the database collation's, case-insensitive on SQL Server — + // so the revision hash of an unchanged configuration is byte-identical across the refactor (fix round 1; + // pinned by ConfigComposerSqlServerHashStabilityTests). Re-sorting in memory with any other comparer would + // re-hash every mixed-case configuration and make drift detection report the whole fleet drifted. private static async Task ReadAsync(OtOpcUaConfigDbContext db, CancellationToken ct) => new DraftSnapshot { @@ -64,26 +70,26 @@ public static class DraftSnapshotFactory // EquipmentUuid checks run in separate validator passes — no rule here reads these fields. GenerationId = 0, // generation model dropped; placeholder (no rule reads it) ClusterId = string.Empty, // global snapshot; rules compare entity ClusterId fields, not this - ServerClusters = await db.ServerClusters.AsNoTracking().ToListAsync(ct), - ClusterNodes = await db.ClusterNodes.AsNoTracking().ToListAsync(ct), + ServerClusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), + ClusterNodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), // v3: Namespace entity retired — no namespace list to materialize. The Raw tree // (RawFolders/TagGroups) + UnsTagReferences feed the v3 rules (charset, tagname length, // effective-leaf uniqueness). - RawFolders = await db.RawFolders.AsNoTracking().ToListAsync(ct), - DriverInstances = await db.DriverInstances.AsNoTracking().ToListAsync(ct), - Devices = await db.Devices.AsNoTracking().ToListAsync(ct), - TagGroups = await db.TagGroups.AsNoTracking().ToListAsync(ct), - UnsAreas = await db.UnsAreas.AsNoTracking().ToListAsync(ct), - UnsLines = await db.UnsLines.AsNoTracking().ToListAsync(ct), - Equipment = await db.Equipment.AsNoTracking().ToListAsync(ct), - Tags = await db.Tags.AsNoTracking().ToListAsync(ct), - UnsTagReferences = await db.UnsTagReferences.AsNoTracking().ToListAsync(ct), - VirtualTags = await db.VirtualTags.AsNoTracking().ToListAsync(ct), - ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().ToListAsync(ct), - Scripts = await db.Scripts.AsNoTracking().ToListAsync(ct), - PollGroups = await db.PollGroups.AsNoTracking().ToListAsync(ct), - NodeAcls = await db.NodeAcls.AsNoTracking().ToListAsync(ct), - ApiKeys = await db.ApiKeys.AsNoTracking().ToListAsync(ct), + RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), + DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), + Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), + TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), + UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), + UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), + Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), + Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), + UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), + VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), + ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), + PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), + NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), PriorEquipment = [], // intentional: no prior-generation table to diff against at this level ActiveReservations = await db.ExternalIdReservations .AsNoTracking() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs index 6f332143..39735ddb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs @@ -9,8 +9,9 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; /// /// Pure snapshot composer: reads the current live-edit state from /// and serialises it into a deterministic, gzip-compressed byte[] artifact + SHA-256 hex revision hash. -/// Determinism comes from sorting every collection by its natural key before serialising, so two snapshots -/// over the same DB state always produce the same hash regardless of EF row ordering; compression wraps the +/// Determinism comes from reading every collection ordered by its natural key on the server +/// (), so two snapshots over the same DB state always produce the same hash +/// — and the same hash as the pre-R03:3.2 composer, which used those same queries; compression wraps the /// final bytes and reorders nothing. The hash is taken over the COMPRESSED bytes — see . /// public static class ConfigComposer @@ -41,6 +42,11 @@ public static class ConfigComposer /// Serialises an already-read into the deterministic artifact. Pure over /// its input: the same snapshot always yields the same bytes and hash, whatever the DB holds by now. /// + /// + /// Serialises the collections IN THE ORDER GIVEN. Determinism comes from + /// , which reads every table ordered by its natural key + /// on the server; a snapshot built any other way must be ordered the same way or its hash will differ. + /// /// The rows to compose — the SAME rows the validator was run over. /// The compressed artifact blob and its revision hash. public static ConfigArtifact Compose(DraftSnapshot snapshot) @@ -52,34 +58,35 @@ public static class ConfigComposer // projection (UnsAreas/UnsLines/Equipment/UnsTagReferences) and the retained VirtualTags/ScriptedAlarms/ // PollGroups/NodeAcls/Scripts. The Namespaces snapshot is RETIRED (the two OPC UA namespaces are implicit). // RevisionHash inputs follow the blob automatically — adding/removing a table shifts the SHA-256. - // Every collection is sorted by its natural key with an ORDINAL comparer so the hash is independent of - // both EF row order and the process culture (a culture-aware sort could order "a-1"/"A-1" differently - // on two central nodes and make the same config hash twice). + // NO re-sort here (fix round 1): the snapshot's collections arrive ordered by their natural key from + // DraftSnapshotFactory's server-side ORDER BY — the same order, and so the same bytes and RevisionHash, + // as the pre-R03:3.2 composer's own queries. An in-memory ordinal re-sort orders mixed-case keys + // differently from the SQL collation and silently re-hashed every unchanged configuration. var ordered = new { - Clusters = snapshot.ServerClusters.OrderBy(x => x.ClusterId, StringComparer.Ordinal).ToList(), - Nodes = snapshot.ClusterNodes.OrderBy(x => x.NodeId, StringComparer.Ordinal).ToList(), - RawFolders = snapshot.RawFolders.OrderBy(x => x.RawFolderId, StringComparer.Ordinal).ToList(), - DriverInstances = snapshot.DriverInstances.OrderBy(x => x.DriverInstanceId, StringComparer.Ordinal).ToList(), - Devices = snapshot.Devices.OrderBy(x => x.DeviceId, StringComparer.Ordinal).ToList(), - TagGroups = snapshot.TagGroups.OrderBy(x => x.TagGroupId, StringComparer.Ordinal).ToList(), - Equipment = snapshot.Equipment.OrderBy(x => x.EquipmentId, StringComparer.Ordinal).ToList(), - Tags = snapshot.Tags.OrderBy(x => x.TagId, StringComparer.Ordinal).ToList(), - UnsTagReferences = snapshot.UnsTagReferences.OrderBy(x => x.UnsTagReferenceId, StringComparer.Ordinal).ToList(), - PollGroups = snapshot.PollGroups.OrderBy(x => x.PollGroupId, StringComparer.Ordinal).ToList(), - UnsAreas = snapshot.UnsAreas.OrderBy(x => x.UnsAreaId, StringComparer.Ordinal).ToList(), - UnsLines = snapshot.UnsLines.OrderBy(x => x.UnsLineId, StringComparer.Ordinal).ToList(), - NodeAcls = snapshot.NodeAcls.OrderBy(x => x.NodeAclId, StringComparer.Ordinal).ToList(), + Clusters = snapshot.ServerClusters, + Nodes = snapshot.ClusterNodes, + RawFolders = snapshot.RawFolders, + DriverInstances = snapshot.DriverInstances, + Devices = snapshot.Devices, + TagGroups = snapshot.TagGroups, + Equipment = snapshot.Equipment, + Tags = snapshot.Tags, + UnsTagReferences = snapshot.UnsTagReferences, + PollGroups = snapshot.PollGroups, + UnsAreas = snapshot.UnsAreas, + UnsLines = snapshot.UnsLines, + NodeAcls = snapshot.NodeAcls, // the data-plane credentials travel with the config, because a driver-only node // has no ConfigDb to read them from (per-cluster mesh Phase 4) and would otherwise deny every // UserName token. Only the peppered HMAC of each secret ships — never the secret — and the // pepper is node configuration that is deliberately NOT in the artifact, so a stolen blob // still cannot be turned into a working credential. This is also why revoking a key reaches a // driver-only node only on the next deployment; see the ApiKey entity's remarks. - ApiKeys = snapshot.ApiKeys.OrderBy(x => x.ApiKeyId, StringComparer.Ordinal).ToList(), - Scripts = snapshot.Scripts.OrderBy(x => x.ScriptId, StringComparer.Ordinal).ToList(), - VirtualTags = snapshot.VirtualTags.OrderBy(x => x.VirtualTagId, StringComparer.Ordinal).ToList(), - ScriptedAlarms = snapshot.ScriptedAlarms.OrderBy(x => x.ScriptedAlarmId, StringComparer.Ordinal).ToList(), + ApiKeys = snapshot.ApiKeys, + Scripts = snapshot.Scripts, + VirtualTags = snapshot.VirtualTags, + ScriptedAlarms = snapshot.ScriptedAlarms, }; return Pack(JsonSerializer.SerializeToUtf8Bytes(ordered, JsonOptions)); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs new file mode 100644 index 00000000..67ac26bd --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs @@ -0,0 +1,87 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Fix round 1 minor — revision-hash stability across the R03:3.2 refactor. The pre-change composer read +/// every table with a SQL ORDER BY (the database's case-INSENSITIVE collation); the refactor re-sorted +/// in memory with an ORDINAL comparer, which orders mixed-case keys differently ("B-1" before "a-2"). The +/// same configuration would then hash differently after the upgrade: drift detection reports every node +/// drifted, and a re-deploy of an unchanged config shows a "new" revision. This pins the composer's output +/// BYTE-identical to a verbatim replica of the pre-change composer on the real provider. +/// +[Trait("Category", "SchemaCompliance")] +public sealed class ConfigComposerSqlServerHashStabilityTests +{ + [Fact] + public async Task Composer_output_is_byte_identical_to_the_pre_change_composer_for_mixed_case_keys() + { + using var sql = new SqlServerTestDatabase(); + await using (var seed = sql.NewContext()) + { + // Collation order a-2, B-1, c-3; ordinal order B-1, a-2, c-3. + foreach (var id in new[] { "c-3", "B-1", "a-2" }) + seed.Scripts.Add(new Script { ScriptId = id, Name = id, SourceCode = "return 1;", SourceHash = id }); + await seed.SaveChangesAsync(); + } + + await using var legacyDb = sql.NewContext(); + var legacy = await LegacySnapshotAndFlattenAsync(legacyDb); + + await using var viaDb = sql.NewContext(); + var current = await ConfigComposer.SnapshotAndFlattenAsync(viaDb); + current.Blob.ShouldBe(legacy.Blob); + current.RevisionHash.ShouldBe(legacy.RevisionHash); + + // The deploy path's form (validate + compose from ONE snapshot) must agree too. + await using var deployDb = sql.NewContext(); + var viaSnapshot = ConfigComposer.Compose(await DraftSnapshotFactory.FromConfigDbAsync(deployDb)); + viaSnapshot.RevisionHash.ShouldBe(legacy.RevisionHash); + } + + /// Verbatim replica of ConfigComposer.SnapshotAndFlattenAsync before R03:3.2 (ad2b5c40). + private static async Task<(byte[] Blob, string RevisionHash)> LegacySnapshotAndFlattenAsync(OtOpcUaConfigDbContext db) + { + var ct = CancellationToken.None; + var snapshot = new + { + Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), + Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), + RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), + DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), + Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), + TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), + Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), + Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), + UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), + PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), + UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), + UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), + NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), + Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), + VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), + ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + }; + var json = JsonSerializer.SerializeToUtf8Bytes(snapshot, new JsonSerializerOptions + { + WriteIndented = false, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never, + }); + using var output = new MemoryStream(json.Length / 8); + using (var gzip = new GZipStream(output, CompressionLevel.Fastest, leaveOpen: true)) + gzip.Write(json, 0, json.Length); + var blob = output.ToArray(); + return (blob, Convert.ToHexStringLower(SHA256.HashData(blob))); + } +} From 896aaa2b02258e4f829f7d5fdff10c7efb57e0b0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:49:56 -0400 Subject: [PATCH 24/27] fix(controlplane): a PSK secret deleted from the store revokes the key; bound refresh concurrency RefreshAsync now tells "the store answered: no such secret" (revocation - drop the refreshed key AND the lazily cached one) apart from "the read failed" (keep the previous key). Store reads fan out through Parallel.ForEachAsync capped at MaxConcurrentRefreshReads = 8 instead of one read per cluster at once. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Communication/IMeshPskProvider.cs | 90 +++++++++++++------ .../MeshPskProviderRefreshTests.cs | 71 +++++++++++++++ 2 files changed, 136 insertions(+), 25 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs index bdab7ba7..bad1956c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs @@ -110,7 +110,8 @@ public interface IMeshPskProvider /// /// Re-reads the keys for from the store OFF the request path and /// swaps them into the cache atomically (R03:3.4). Until the refresh completes, lookups keep - /// serving the previous key; a cluster whose read fails keeps its previous key too. Replaces + /// serving the previous key; a cluster whose read FAILS keeps its previous key too, while a cluster whose + /// secret the store reports ABSENT is revoked (no key from then on). Replaces /// "invalidate on every refresh", which made the first inbound ack per cluster per /// ContactRefreshSeconds block a gRPC thread on a synchronous, 3 s-bounded store read. /// @@ -331,45 +332,76 @@ public sealed class MeshPskProvider : IMeshPskProvider _refreshed = new Dictionary(StringComparer.OrdinalIgnoreCase); } + /// + /// Ceiling on concurrent store reads during one . A fleet of N clusters used to + /// fan out N simultaneous reads every contact refresh; the store (and anything behind it) sees at most + /// this many at once now. + /// + public const int MaxConcurrentRefreshReads = 8; + /// public async Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) { ArgumentNullException.ThrowIfNull(clusterIds); var previous = _refreshed; - var next = new Dictionary(StringComparer.OrdinalIgnoreCase); var ids = clusterIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + var results = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); - // Resolve every cluster concurrently — each read bounded like the synchronous path — and carry a - // failed cluster's PREVIOUS refreshed key forward: a transient store fault must not turn into a - // window in which that cluster has no key at all. - var reads = ids.Select(async id => + // Bounded fan-out; each read bounded like the synchronous path. + await Parallel.ForEachAsync( + ids, + new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRefreshReads, CancellationToken = ct }, + async (id, token) => results[id] = await ResolveFromStoreAsync(id, token).ConfigureAwait(false)) + .ConfigureAwait(false); + + // Three outcomes per cluster (fix round 1 / I8): + // - Found → the new key; + // - NotFound → the store answered that the secret does not exist: a REVOCATION — the cluster gets no + // key (and its lazily cached one is evicted below), so a deleted secret stops working + // within one refresh instead of living on for the life of the process; + // - Failed → the store could not be read (threw / timed out / unnameable): keep the PREVIOUS key — + // a transient fault must not open a window in which the cluster has no key at all. + var next = new Dictionary(StringComparer.OrdinalIgnoreCase); + var answered = new List(); + foreach (var id in ids) { - var value = await ResolveFromStoreAsync(id, ct).ConfigureAwait(false); - return (Id: id, Value: value); - }).ToList(); - foreach (var (id, value) in await Task.WhenAll(reads).ConfigureAwait(false)) - { - if (!string.IsNullOrWhiteSpace(value)) next[id] = value; - else if (previous.TryGetValue(id, out var kept)) next[id] = kept; + var read = results[id]; + switch (read.Outcome) + { + case StoreReadOutcome.Found: + next[id] = read.Value!; + answered.Add(id); + break; + case StoreReadOutcome.NotFound: + answered.Add(id); + break; + default: + if (previous.TryGetValue(id, out var kept)) next[id] = kept; + break; + } } - // The swap. Then drop the lazy entries for the clusters now covered, so a key rotated between - // refreshes is served from the new snapshot rather than from a stale lazy resolution; entries - // for clusters NOT covered (their read failed and they had no previous value) are left alone. + // The swap. Then drop the lazy entries for every cluster the store ANSWERED for — found (serve the new + // snapshot, not a stale lazy resolution) or not found (revoked: a lazily cached key must die too). + // Entries for clusters whose read failed are left alone. _refreshed = next; - foreach (var id in next.Keys) _cache.TryRemove(id, out _); + foreach (var id in answered) _cache.TryRemove(id, out _); } + private enum StoreReadOutcome { Found, NotFound, Failed } + + private readonly record struct StoreRead(StoreReadOutcome Outcome, string? Value); + /// - /// The asynchronous twin of : same name derivation, same bound, same - /// "anything but a usable key is null" contract — awaited instead of blocked on, because this one - /// runs on a refresh rather than inside a gRPC interceptor. + /// The asynchronous twin of : same name derivation, same bound — awaited + /// instead of blocked on, because this one runs on a refresh rather than inside a gRPC interceptor — + /// but it keeps "the secret does not exist" (a blank answer) apart from "the store could not be read". /// /// The application Cluster id. /// The caller's token, linked under the store-read bound. - /// The stored key, or null. - private async Task ResolveFromStoreAsync(string clusterId, CancellationToken ct) + /// The outcome and, when found, the key. + private async Task ResolveFromStoreAsync(string clusterId, CancellationToken ct) { SecretName name; try @@ -384,14 +416,22 @@ public sealed class MeshPskProvider : IMeshPskProvider + "refreshed from the store. Supply it through MeshTransport:ClusterPsks instead, or " + "rename the cluster to [a-z0-9._/-].", clusterId); - return null; + return new StoreRead(StoreReadOutcome.Failed, null); } using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); cts.CancelAfter(StoreReadTimeout); try { - return await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false); + var value = await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(value)) return new StoreRead(StoreReadOutcome.Found, value); + + _logger.LogWarning( + "Mesh PSK secret {SecretName} for cluster {ClusterId} is absent (or blank) in the store; the key is " + + "REVOKED — commands to that cluster and acks from it are refused until it is restored.", + name.Value, + clusterId); + return new StoreRead(StoreReadOutcome.NotFound, null); } catch (Exception ex) { @@ -401,7 +441,7 @@ public sealed class MeshPskProvider : IMeshPskProvider + "previously resolved key (if any) stays in service until the next refresh.", clusterId, name.Value); - return null; + return new StoreRead(StoreReadOutcome.Failed, null); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs index e727c153..34129235 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs @@ -78,6 +78,77 @@ public sealed class MeshPskProviderRefreshTests psk.ShouldBe("k-1"); } + /// + /// Fix round 1 / I8 — "the store says there is no such secret" is a REVOCATION, not a read failure: + /// the previous key must stop being served (from the refreshed snapshot AND the lazy cache). A thrown + /// read keeps the previous key (the test above); a definitive not-found drops it. + /// + [Fact] + public async Task A_secret_deleted_from_the_store_revokes_the_key_on_the_next_refresh() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + provider.Matches("SITE-A", "k-1").ShouldBeTrue("positive control: the key was in service"); + + resolver.Value = null; // the secret was deleted + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); + provider.Matches("SITE-A", "k-1").ShouldBeFalse(); + } + + /// The same revocation reaches a key that was first served from the LAZY cache, not a refresh. + [Fact] + public async Task A_deleted_secret_also_evicts_a_lazily_cached_key() + { + var resolver = new GatedResolver { Value = "k-lazy" }; + var provider = Provider(resolver); + provider.TryGetPsk("SITE-A", out var lazy).ShouldBeTrue(); + lazy.ShouldBe("k-lazy"); + + resolver.Value = null; + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); + } + + /// Fix round 1 minor — the refresh fans out to the store with BOUNDED concurrency, not one read per cluster at once. + [Fact] + public async Task Refresh_store_reads_are_concurrency_bounded() + { + var resolver = new ConcurrencyTrackingResolver(); + var provider = Provider(resolver); + var clusters = Enumerable.Range(0, 40).Select(i => $"SITE-{i}").ToList(); + + await provider.RefreshAsync(clusters, CancellationToken.None); + + resolver.Calls.ShouldBe(40); + resolver.MaxConcurrent.ShouldBeLessThanOrEqualTo(MeshPskProvider.MaxConcurrentRefreshReads); + resolver.MaxConcurrent.ShouldBeGreaterThan(1, "still concurrent, just bounded"); + foreach (var c in clusters) provider.TryGetPsk(c, out _).ShouldBeTrue(); + } + + private sealed class ConcurrencyTrackingResolver : ISecretResolver + { + private int _inFlight; + private int _max; + private int _calls; + public int MaxConcurrent => Volatile.Read(ref _max); + public int Calls => Volatile.Read(ref _calls); + + public async Task GetAsync(SecretName name, CancellationToken ct) + { + Interlocked.Increment(ref _calls); + var now = Interlocked.Increment(ref _inFlight); + int seen; + while (now > (seen = Volatile.Read(ref _max)) && Interlocked.CompareExchange(ref _max, now, seen) != seen) { } + await Task.Delay(20, ct); + Interlocked.Decrement(ref _inFlight); + return "k-" + name.Value; + } + } + /// A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour). [Fact] public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily() From dc3323fcca2d6a213fc63d5b1c95049015bd17ff Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:51:28 -0400 Subject: [PATCH 25/27] fix(host): publish the known-id set and its stamp atomically; never write the shared guard options BoundedApiKeyStore kept the set (volatile) and its DateTimeOffset stamp (non-volatile, 16 bytes) in separate fields, so a reader could pair a new set with an old stamp or read a torn stamp; they are now one immutable snapshot behind a volatile reference. An inner store that cannot list ids now switches the set off with a per-instance flag instead of writing KnownIdSet = false into the shared ApiKeyStoreGuardOptions. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../OpcUa/BoundedApiKeyStore.cs | 35 ++++++++++++------- .../OpcUa/BoundedApiKeyStoreKnownIdTests.cs | 30 ++++++++++++++++ 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index bb3154ed..6ebb7626 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -61,10 +61,22 @@ public sealed class BoundedApiKeyStore : IApiKeyStore // R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds // (single-flight; a refresh that fails or times out keeps the previous set). Null until the first // successful load, or when the inner store cannot list ids — then the per-id path runs as before. - private volatile IReadOnlySet? _knownIds; - private DateTimeOffset _knownIdsStampedUtc = DateTimeOffset.MinValue; + // The set and the instant it was loaded are published TOGETHER as one immutable snapshot behind a volatile + // reference (fix round 1): a reader on another thread can never pair a new set with an old stamp, or read + // a torn DateTimeOffset (a 16-byte struct, not atomically readable), as the separate fields allowed. + private volatile KnownIdSnapshot? _knownIds; private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1); + // Set when the inner store reports it cannot list ids. Per-instance on purpose: the guard options object is + // shared configuration and is never written (it used to be — `_guard.KnownIdSet = false` — which leaked + // one store's capability into every other consumer of the same options). + private volatile bool _listingUnsupported; + + /// The known-id set and the instant it was loaded, published as one unit. + /// The ids the inner store holds. + /// When they were read. + private sealed record KnownIdSnapshot(IReadOnlySet Ids, DateTimeOffset StampedUtc); + // R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards // the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the // configured rate instead of one SQL round trip per attempt under the SessionManager lock. @@ -104,7 +116,7 @@ public sealed class BoundedApiKeyStore : IApiKeyStore } /// The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards. - public IReadOnlySet? KnownKeyIds => _knownIds; + public IReadOnlySet? KnownKeyIds => _knownIds?.Ids; /// The store being bounded. /// @@ -173,26 +185,25 @@ public sealed class BoundedApiKeyStore : IApiKeyStore /// The set, or null. private async Task?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct) { - if (!_guard.KnownIdSet || ttl <= TimeSpan.Zero) return null; + if (!_guard.KnownIdSet || _listingUnsupported || ttl <= TimeSpan.Zero) return null; var now = _time.GetUtcNow(); var current = _knownIds; - if (current is not null && now - _knownIdsStampedUtc < ttl) return current; + if (current is not null && now - current.StampedUtc < ttl) return current.Ids; - if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current; + if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current?.Ids; try { // Re-check under the gate: another caller may have refreshed while this one waited. - if (_knownIds is { } refreshed && now - _knownIdsStampedUtc < ttl) return refreshed; + if (_knownIds is { } refreshed && now - refreshed.StampedUtc < ttl) return refreshed.Ids; var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false); if (ids is null) { - // Unsupported by the inner store: say so once and stop asking. - _guard.KnownIdSet = false; + // Unsupported by the inner store: stop asking — for this decorator only. + _listingUnsupported = true; return null; } - _knownIds = ids; - _knownIdsStampedUtc = _time.GetUtcNow(); + _knownIds = new KnownIdSnapshot(ids, _time.GetUtcNow()); return ids; } catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) @@ -200,7 +211,7 @@ public sealed class BoundedApiKeyStore : IApiKeyStore _log.LogWarning(ex, "ApiKeyStore: refreshing the known key-id set failed; {Mode}", current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force"); - return current; + return current?.Ids; } finally { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs index 7051b151..8d08df5e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs @@ -120,6 +120,36 @@ public sealed class BoundedApiKeyStoreKnownIdTests inner.ListCalls.ShouldBe(0); } + /// + /// Fix round 1 minor — an inner store that cannot list ids switches the known-id set off for THIS + /// decorator only. It used to do so by writing KnownIdSet = false into the guard options it was + /// handed, a shared options object, so one store's capability leaked into every other consumer. + /// Positive control: a second decorator over a listing store on the same options instance still uses + /// the set. + /// + [Fact] + public async Task An_unlisting_store_does_not_mutate_the_shared_guard_options() + { + var guard = new ApiKeyStoreGuardOptions { MissBurst = 100, MissRefillPerSecond = 20 }; + var options = Options.Create(new ApiKeyOptions + { + Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = 5, + }); + var clock = new FakeClock(T0); + var unlisting = new ListingStore("k-real") { SupportsListing = false }; + var first = new BoundedApiKeyStore(unlisting, options, NullLogger.Instance, clock, guard); + + (await first.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + unlisting.FindCalls.ShouldBe(1, "no set: the per-id path ran"); + guard.KnownIdSet.ShouldBeTrue("the shared options object is never written"); + + var listing = new ListingStore("k-real"); + var second = new BoundedApiKeyStore(listing, options, NullLogger.Instance, clock, guard); + (await second.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + listing.FindCalls.ShouldBe(0, "the second decorator still refuses unknown ids in memory"); + listing.ListCalls.ShouldBe(1); + } + private static BoundedApiKeyStore Bounded( IApiKeyStore inner, TimeProvider clock, int cacheSeconds, int missBurst = 100, double refillPerSecond = 20) => new( inner, From a85d1c069a477abb308051303c1fb0a9ccf62b4b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:59:10 -0400 Subject: [PATCH 26/27] fix(scripting): quarantine a refusal behind a stuck orphan (fix round 2) Ruling refinement on I1. Only timeouts are strikes, so a never-returning while(true), which times out once and is then only refused, was never quarantined, and that defeated E1. A refusal now quarantines immediately when the orphan it was refused behind has been executing for longer than ScriptingOptions.StuckOrphanQuarantineFactor x the script's timeout (default 10, so 20 s at the 2 s default; 0 or less disables it). It uses the same quarantine path: one Error, otopcua.script.quarantined with reason=stuck, Bad, lifted on source change or redeploy. A young orphan (a short overrun) is still a plain refusal. The guard stamps the run's start (Stopwatch timestamp) when the pool picks it up and clears it when the run really finishes. Tests: the 30-trigger while(true) regression (paced) now ends quarantined with reason "stuck", still with exactly one orphan and a healthy sibling. Positive control: a 2x overrun is refused, not quarantined, and evaluates again once the orphan completes. A factor of 0 disables the rule. Docs (ScriptEditor "Runtime safety", VirtualTags) and the seam remarks are updated. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/ScriptEditor.md | 11 ++- docs/VirtualTags.md | 2 +- .../ScriptedAlarmEngine.cs | 2 +- .../ScriptRunGuard.cs | 36 +++++++- .../ScriptingOptions.cs | 13 +++ .../CalculationEvaluator.cs | 2 +- .../Engines/RoslynVirtualTagEvaluator.cs | 2 +- .../ScriptRunGuardTests.cs | 88 ++++++++++++++++--- 8 files changed, 133 insertions(+), 23 deletions(-) diff --git a/docs/ScriptEditor.md b/docs/ScriptEditor.md index 41421d34..eb6985e2 100644 --- a/docs/ScriptEditor.md +++ b/docs/ScriptEditor.md @@ -190,9 +190,14 @@ scripted-alarm predicates) runs scripts through the shared `ScriptRunGuard`: Only a timeout is a strike: a refusal neither adds to nor resets the streak, and a run that returns (value or throw) resets it. A quarantined script is never dispatched again; it reads Bad `"script quarantined: …"`, logs one Error and - increments `otopcua_script_quarantined`. An infinite loop therefore usually shows - as *refused* (one timeout, then refusals for as long as the orphan spins) rather - than *quarantined*. + increments `otopcua_script_quarantined`. +- **Stuck-orphan quarantine.** A never-returning script (`while(true)`) times out + once and is then only refused, so the timeout streak alone never reaches 3. + Instead, a refusal behind an orphan that has been running longer than + `Scripting:StuckOrphanQuarantineFactor` × the timeout (default 10 × — 20 s at the + 2 s default) quarantines immediately, through the same path (Error log, metric + `reason=stuck`, Bad). A short overrun (a young orphan) stays a plain refusal and + the script resumes once the orphan completes. - **Allocation budget.** A single run allocating more than `Scripting:MaxAllocBytesPerRun` (64 MB) is quarantined. This is sampled after the run returns, so a run that OOMs the process first is not stopped — that needs an diff --git a/docs/VirtualTags.md b/docs/VirtualTags.md index 50ef578f..183fcef0 100644 --- a/docs/VirtualTags.md +++ b/docs/VirtualTags.md @@ -34,7 +34,7 @@ User scripts are compiled via `Microsoft.CodeAnalysis.CSharp` (regular compiler, #### Known resource limits (accepted trade-offs) -The sandbox cannot pre-empt a script. **Memory:** each run's own-thread allocation is sampled (`GC.GetAllocatedBytesForCurrentThread`) and a run over `Scripting:MaxAllocBytesPerRun` (default 64 MB) quarantines the script — but the sample is taken after the run returns, so a script that allocates fast enough (`new byte[int.MaxValue]` in a loop) can still OOM the process first. **CPU:** a tight loop keeps its pool thread after the timeout fires; `ScriptRunGuard` bounds that to one orphan per script by refusing further runs while it spins (Bad `"previous evaluation still running"`) and quarantines after three consecutive timeouts (a refusal is not a strike). Both are mitigations; pre-emptive CPU/memory limits need an out-of-process runner. Script authoring is gated behind the Admin permission as the primary control. See `docs/ScriptEditor.md` §"Runtime safety". +The sandbox cannot pre-empt a script. **Memory:** each run's own-thread allocation is sampled (`GC.GetAllocatedBytesForCurrentThread`) and a run over `Scripting:MaxAllocBytesPerRun` (default 64 MB) quarantines the script — but the sample is taken after the run returns, so a script that allocates fast enough (`new byte[int.MaxValue]` in a loop) can still OOM the process first. **CPU:** a tight loop keeps its pool thread after the timeout fires; `ScriptRunGuard` bounds that to one orphan per script by refusing further runs while it spins (Bad `"previous evaluation still running"`) and quarantines after three consecutive timeouts (a refusal is not a strike) — or immediately on a refusal behind an orphan that has run longer than `Scripting:StuckOrphanQuarantineFactor` (default 10) × the timeout, which is what quarantines a never-returning `while(true)`. Both are mitigations; pre-emptive CPU/memory limits need an out-of-process runner. Script authoring is gated behind the Admin permission as the primary control. See `docs/ScriptEditor.md` §"Runtime safety". `System.Threading.Tasks` is denied (Core.Scripting-003), which prevents `Task.Run` / `Parallel` fan-out that would spawn background work outliving the timeout. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index c96c8c87..17b513ac 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -76,7 +76,7 @@ public sealed class ScriptedAlarmEngine : IDisposable /// /// Per-alarm run quarantine (keyed by alarm id, source identity = predicate source). Refuses a /// predicate run while the alarm's previous (timed-out, abandoned) run is still executing and - /// quarantines after three consecutive timeouts (a refusal is not a strike) or an allocation-budget breach — one orphan + /// quarantines after three consecutive timeouts (a refusal is not a strike unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach — one orphan /// per alarm instead of one per upstream push. Reset on every (a redeploy). /// private readonly ScriptRunGuard _guard; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs index dc308ce6..de1afee2 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Diagnostics; using System.Diagnostics.Metrics; using Serilog; using ZB.MOM.WW.OtOpcUa.Commons.Observability; @@ -27,7 +28,10 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// consecutive timeouts (only a timeout is a strike; a refusal is a Bad publish plus its counter /// and leaves the streak untouched — on a 1 Hz trigger with a 2 s budget one 4 s overrun is a /// timeout plus two refusals and must not quarantine; a run that returns — a value or a throw — -/// resets the streak): no further dispatch, every attempt throws +/// resets the streak), or on a refusal behind a stuck orphan — one that has been executing for +/// longer than × the timeout (20 s at the +/// 2 s default), which is how a never-returning while(true) (one timeout, then only refusals) +/// gets quarantined: no further dispatch, every attempt throws /// , one Error log, one otopcua.script.quarantined /// increment, one event. Lifted by a source change (the operator fixed the /// script — a new source identity under the same key starts clean and the old orphan no longer blocks @@ -63,11 +67,11 @@ public sealed class ScriptRunGuard /// /// Counter otopcua.script.quarantined (Prometheus: otopcua_script_quarantined): one - /// increment per script quarantined, tagged reason = timeouts | allocation. + /// increment per script quarantined, tagged reason = timeouts | stuck | allocation. /// public static readonly Counter QuarantinedCounter = OtOpcUaTelemetry.Meter.CreateCounter("otopcua.script.quarantined", unit: "{script}", - description: "Scripts quarantined by the run guard (reason: timeouts | allocation)."); + description: "Scripts quarantined by the run guard (reason: timeouts | stuck | allocation)."); /// /// Counter otopcua.script.run_refused: runs refused because the script's previous run was @@ -80,6 +84,7 @@ public sealed class ScriptRunGuard private readonly ConcurrentDictionary _keys = new(StringComparer.Ordinal); private readonly long _maxAllocBytesPerRun; private readonly int _quarantineThreshold; + private readonly double _stuckFactor; private readonly ILogger _logger; private int _inFlight; private long _started; @@ -94,6 +99,7 @@ public sealed class ScriptRunGuard options ??= new ScriptingOptions(); _maxAllocBytesPerRun = options.MaxAllocBytesPerRun; _quarantineThreshold = Math.Max(1, options.QuarantineAfterConsecutiveTimeouts); + _stuckFactor = options.StuckOrphanQuarantineFactor; _logger = logger ?? Log.Logger; } @@ -185,6 +191,24 @@ public sealed class ScriptRunGuard // ran long once. The refusal also leaves the timeout streak untouched. Interlocked.Increment(ref _refused); RefusedCounter.Add(1, new KeyValuePair("script", key)); + // …unless the orphan is STUCK: a never-returning script times out once and is then only ever + // refused, so the timeout streak alone would never quarantine it. Past the stuck threshold the + // refusal itself quarantines (same path: Error log, metric, Bad, lifted on redeploy/source change). + if (_stuckFactor > 0 && Volatile.Read(ref state.RunStartedTimestamp) is var startedAt and not 0) + { + var running = Stopwatch.GetElapsedTime(startedAt); + var threshold = timeout * _stuckFactor; + if (running > threshold) + { + Quarantine(key, state, + $"its previous evaluation has been running for {running.TotalSeconds:F1} s, over " + + $"{_stuckFactor:0.##}x its {timeout.TotalMilliseconds:F0} ms budget " + + "(Scripting:StuckOrphanQuarantineFactor) — it is stuck; the script is not evaluated again " + + "until its source changes or it is redeployed", + "stuck"); + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + } + } throw new ScriptRunRefusedException(key); } @@ -200,6 +224,7 @@ public sealed class ScriptRunGuard // never complete, and the slot would never be released. The evaluator observes the token instead. var runTask = Task.Run(async () => { + Volatile.Write(ref state.RunStartedTimestamp, Stopwatch.GetTimestamp()); started.TrySetResult(); var before = GC.GetAllocatedBytesForCurrentThread(); try @@ -223,6 +248,7 @@ public sealed class ScriptRunGuard // still quarantines its key here. _ = runTask.ContinueWith(t => { + Volatile.Write(ref state.RunStartedTimestamp, 0); Interlocked.Exchange(ref state.InFlight, 0); Interlocked.Decrement(ref _inFlight); runCts.Dispose(); @@ -339,6 +365,8 @@ public sealed class ScriptRunGuard public string? SourceIdentity = sourceIdentity; public int InFlight; public int ConsecutiveTimeouts; + /// when the pool started the in-flight run; 0 when idle. + public long RunStartedTimestamp; public string? QuarantineReason; } @@ -373,7 +401,7 @@ public sealed class ScriptRunRefusedException : Exception /// /// Thrown by for a quarantined script: repeated -/// timeouts or an allocation-budget breach. Nothing is dispatched while quarantined; the seam +/// timeouts, a stuck orphan or an allocation-budget breach. Nothing is dispatched while quarantined; the seam /// publishes Bad carrying the reason, which an operator clears by fixing the script (source change) or /// redeploying. /// diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs index f101a0e1..296d862c 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs @@ -25,6 +25,9 @@ public sealed class ScriptingOptions /// Default : 3. public const int DefaultQuarantineAfterConsecutiveTimeouts = 3; + /// Default : 10. + public const double DefaultStuckOrphanQuarantineFactor = 10; + /// /// Managed bytes one script run may allocate on its own thread (sampled with /// around the run) before the script is @@ -40,4 +43,14 @@ public sealed class ScriptingOptions /// throw) resets it. Default 3; values below 1 are treated as 1. /// public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts; + + /// + /// A refusal quarantines the script immediately when the abandoned run it was refused behind has been + /// executing for longer than this many times the script's timeout (20 s at the 2 s default). This is + /// what quarantines a permanently stuck script (while(true)): it times out once and is then + /// only ever refused, so the timeout streak alone would never reach + /// . A young orphan (a short overrun) is still just a + /// refusal. Zero or negative disables the rule. Default 10. + /// + public double StuckOrphanQuarantineFactor { get; set; } = DefaultStuckOrphanQuarantineFactor; } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs index e47cc745..dd7085e3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs @@ -42,7 +42,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// Run quarantine. Every compiled run goes through one keyed by /// calc-tag RawPath with the script source as the identity: a run is refused while the tag's previous /// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive -/// timeouts (a refusal is Bad but not a strike) or an allocation-budget breach — both surface as a +/// timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach — both surface as a /// the driver maps to Bad. The driver's evaluation pump awaits rather than /// blocking a pool thread on the sync , /// and the guard starts the budget clock only once the pool has picked the script up, so pool starvation diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs index f0bbd150..8a9347d9 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs @@ -35,7 +35,7 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines; /// Run quarantine. Every compiled run goes through one keyed by /// virtual-tag id with the expression as the source identity: a run is refused while the tag's previous /// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive -/// timeouts (a refusal is Bad but not a strike) or an allocation-budget breach. Both surface as +/// timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach. Both surface as /// so VirtualTagActor publishes Bad exactly as it does for a timeout. A quarantine lifts when the /// expression changes, on / (the deploy /// boundary), never by itself. diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs index f60045b9..b4efe8af 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs @@ -151,10 +151,13 @@ public sealed class ScriptRunGuardTests } /// - /// The thread-leak regression. Pre-guard, a while(true) script on a 1 Hz trigger leaked one - /// pool thread per trigger — 30 orphans after 30 triggers, trending toward pool starvation. Now the - /// first trigger's orphan is the only one, every later trigger is refused, and a healthy - /// sibling keeps evaluating throughout (the positive control an absence assertion needs). + /// The thread-leak regression, and the stuck-orphan quarantine (fix round 2). Pre-guard, a + /// while(true) script on a 1 Hz trigger leaked one pool thread per trigger — 30 orphans after 30 + /// triggers, trending toward pool starvation. Now the first trigger's orphan is the only one; later + /// triggers are refused while it is young, and once it has run longer than + /// × the timeout (10 × 60 ms here — the + /// trigger cadence is scaled down from 1 Hz to keep the test short) the next refusal quarantines it. + /// A healthy sibling keeps evaluating throughout (the positive control an absence assertion needs). /// /// /// The load-bearing assertions are on the guard's own bookkeeping ( @@ -163,30 +166,37 @@ public sealed class ScriptRunGuardTests /// it must stay far below the 30 the leak would have produced. /// [Fact] - public async Task Runaway_script_leaves_at_most_one_orphan_after_30_triggers_while_a_sibling_keeps_evaluating() + public async Task Runaway_script_leaves_one_orphan_is_quarantined_once_stuck_and_a_sibling_keeps_evaluating() { var guard = new ScriptRunGuard(); using var spin = Compile(SpinScript); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; var threadsBefore = ThreadPool.ThreadCount; + var stuckAfter = ShortTimeout * ScriptingOptions.DefaultStuckOrphanQuarantineFactor; // 600 ms var healthyResults = new List(); + var outcomes = new List(); + var sw = System.Diagnostics.Stopwatch.StartNew(); for (var trigger = 0; trigger < 30; trigger++) { - try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); } - catch (ScriptTimeoutException) { } - catch (ScriptRunRefusedException) { } - catch (ScriptQuarantinedException) { } + try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); outcomes.Add("value"); } + catch (ScriptTimeoutException) { outcomes.Add("timeout"); } + catch (ScriptRunRefusedException) { outcomes.Add(sw.Elapsed < stuckAfter ? "refused-young" : "refused"); } + catch (ScriptQuarantinedException) { outcomes.Add("quarantined"); } healthyResults.Add(await Run(guard, "healthy", FastScript, fast, ct: ct)); + await Task.Delay(50, ct); } guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing"); guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling"); - // One timeout + 29 refusals: not quarantined (a refusal is not a strike), but never re-dispatched. - guard.IsQuarantined("runaway", out _).ShouldBeFalse(); - guard.RefusedWhileRunningCount.ShouldBe(29); + outcomes[0].ShouldBe("timeout"); + outcomes.ShouldContain("refused-young", "a young orphan is only refused — a short overrun is not quarantined"); + guard.IsQuarantined("runaway", out var reason).ShouldBeTrue("the orphan outlived 10x its budget: it is stuck"); + reason!.ShouldContain("stuck"); + outcomes[^1].ShouldBe("quarantined"); + guard.QuarantinedCount.ShouldBe(1); healthyResults.ShouldAllBe(v => v == 42); healthyResults.Count.ShouldBe(30); @@ -194,6 +204,60 @@ public sealed class ScriptRunGuardTests threadDelta.ShouldBeLessThan(15, "thirty leaked orphans would have grown the pool by ~30 threads"); } + /// + /// Positive control for the stuck rule: a 2x overrun (one timeout, refusals while the orphan is young) + /// is NOT quarantined, and once the orphan completes the script evaluates normally again. + /// + [Fact] + public async Task A_2x_overrun_is_not_quarantined_and_resumes_evaluating_once_the_orphan_completes() + { + var guard = new ScriptRunGuard(); + const string source = + """ + var ms = (int)(ctx.GetTag("spinMs").Value ?? 0); + var end = System.DateTime.UtcNow.AddMilliseconds(ms); + while (System.DateTime.UtcNow < end) { } + return 7; + """; + using var eval = Compile(source); + var ct = TestContext.Current.CancellationToken; + var timeout = TimeSpan.FromMilliseconds(100); + + FakeScriptContext Ctx(int spinMs) + { + var c = new FakeScriptContext(); + c.Tags["spinMs"] = new Core.Abstractions.DataValueSnapshot(spinMs, 0u, c.Now, c.Now); + return c; + } + + // Warm the JIT so the first measured run's overrun is the script's, not the compiler's. + (await guard.RunAsync("k", source, eval, Ctx(0), TimeSpan.FromSeconds(5), ct)).ShouldBe(7); + + // 2x overrun: 200 ms against a 100 ms budget — times out, then is refused while it finishes. + await Should.ThrowAsync(() => guard.RunAsync("k", source, eval, Ctx(200), timeout, ct)); + await Should.ThrowAsync(() => guard.RunAsync("k", source, eval, Ctx(0), timeout, ct)); + await WaitForIdle(guard); + + guard.IsQuarantined("k", out _).ShouldBeFalse("a 2x overrun is far below the 10x stuck threshold"); + (await guard.RunAsync("k", source, eval, Ctx(0), timeout, ct)).ShouldBe(7, "the script resumes evaluating"); + guard.QuarantinedCount.ShouldBe(0); + } + + /// The stuck rule is a knob: with ≤ 0 a + /// stuck orphan is only ever refused. + [Fact] + public async Task StuckOrphanQuarantineFactor_zero_disables_the_stuck_rule() + { + var guard = new ScriptRunGuard(new ScriptingOptions { StuckOrphanQuarantineFactor = 0 }); + using var spin = Compile(SpinScript); + var ct = TestContext.Current.CancellationToken; + + await Should.ThrowAsync(() => Run(guard, "k", SpinScript, spin, ShortTimeout, ct)); + await Task.Delay(ShortTimeout * 12, ct); + await Should.ThrowAsync(() => Run(guard, "k", SpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeFalse(); + } + [Fact] public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot() { From 5097444d56f9b7cad8ae979eba5c17fd35ed2c41 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:08:42 -0400 Subject: [PATCH 27/27] fix(controlplane): condition the dispatch flip on the guard's RowVersion; one deadline source; align IdentityModel pins Re-review 1: HandleDispatch's terminal guard and its AwaitingApplyAcks flip read the row separately, so a lost-dispatch closure committed between them was silently reopened. The flip now carries the RowVersion the guard read; a change in between is a concurrency conflict (nothing seeded, nothing published, coordinator reset). Pinned on real SQL Server with a closure injected between the two reads (fails without the fix). AdminOps' apply deadline and the coordinator's now both read DeployTimings.ApplyDeadline, passed explicitly to both singletons. Microsoft.IdentityModel.Protocols(.OpenIdConnect) pinned to 8.11.0 and referenced directly by the Security project, so the whole IdentityModel family resolves 8.11.0 (fresh restore clean of NU190x/NU1605). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- Directory.Packages.props | 8 ++- .../AdminOperations/AdminOperationsActor.cs | 9 +-- .../Coordinators/ConfigPublishCoordinator.cs | 63 +++++++++++----- .../ServiceCollectionExtensions.cs | 8 ++- .../ZB.MOM.WW.OtOpcUa.Security.csproj | 3 + .../ConfigPublishCoordinatorRecoveryTests.cs | 71 +++++++++++++++++++ 6 files changed, 138 insertions(+), 24 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index ed2dca67..d63e819e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -36,8 +36,10 @@ + line as the other Microsoft.AspNetCore.* pins. It depends on Microsoft.IdentityModel.Protocols(.OpenIdConnect) + 8.0.1; those two are pinned to 8.11.0 below (and referenced directly by the Security project, the JwtBearer + entry point, so the pin applies without transitive pinning) to keep the whole IdentityModel family on one + version with Microsoft.IdentityModel.Tokens / System.IdentityModel.Tokens.Jwt. --> @@ -86,6 +88,8 @@ + + diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index 81f09dae..69e878db 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -11,7 +11,6 @@ using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Configuration.Validation; -using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Scripting; @@ -55,7 +54,7 @@ public sealed class AdminOperationsActor : ReceiveActor /// Central comm actor commands are handed to, or to publish on the /// DistributedPubSub mediator directly (the behaviour before the per-cluster mesh). /// - /// The coordinator's apply deadline; defaults to . + /// The coordinator's apply deadline; defaults to (the coordinator's default source). /// Props configured to create an AdminOperationsActor. public static Props Props( IDbContextFactory dbFactory, @@ -73,7 +72,7 @@ public sealed class AdminOperationsActor : ReceiveActor /// Driver probes registered in DI; keyed by DriverType (case-insensitive). /// Deploy-gate TagConfig strictness mode (Warn default | Error opt-in). /// Central comm actor, or for the mediator. - /// The coordinator's apply deadline; defaults to . + /// The coordinator's apply deadline; defaults to (the coordinator's default source). public AdminOperationsActor( IDbContextFactory dbFactory, IActorRef coordinator, @@ -82,7 +81,9 @@ public sealed class AdminOperationsActor : ReceiveActor IActorRef? meshRouter = null, TimeSpan? applyDeadline = null) { - _applyDeadline = applyDeadline ?? ConfigPublishCoordinator.DefaultApplyDeadline; + // The same source the coordinator's default reads (ConfigPublishCoordinator.DefaultApplyDeadline IS + // DeployTimings.ApplyDeadline), so the lost-dispatch closure and the coordinator's own deadline agree. + _applyDeadline = applyDeadline ?? DeployTimings.ApplyDeadline; _dbFactory = dbFactory; _coordinator = coordinator; _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index 49b54df4..d408343c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -237,21 +237,28 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it. + // + // The RowVersion read HERE is the one the AwaitingApplyAcks flip below is conditioned on (re-review 1): + // reading a fresh one at flip time left a window in which AdminOps' lost-dispatch closure could commit + // TimedOut between the guard and the flip, and the flip would silently reopen the row. With the guard's + // RowVersion, any change since the guard read makes the flip a concurrency conflict instead. + byte[]? guardRowVersion; using (var check = _dbFactory.CreateDbContext()) { - var status = check.Deployments + var guard = check.Deployments .AsNoTracking() .Where(d => d.DeploymentId == msg.DeploymentId.Value) - .Select(d => (DeploymentStatus?)d.Status) + .Select(d => new { d.Status, d.RowVersion }) .FirstOrDefault(); - if (status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) + if (guard?.Status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) { _log.Warning( "Ignoring DispatchDeployment {Id}: the deployment is already {Status} (a late or duplicated dispatch)", - msg.DeploymentId, status); + msg.DeploymentId, guard.Status); if (_current == msg.DeploymentId) ResetForNext(); return; } + guardRowVersion = guard?.RowVersion; } _current = msg.DeploymentId; @@ -290,8 +297,22 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers }); } } - UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks); - db.SaveChanges(); + UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks, rowVersion: guardRowVersion); + try + { + db.SaveChanges(); + } + catch (DbUpdateConcurrencyException) + { + // The row moved after the guard read (on SQL Server: AdminOps closed it as a lost dispatch). The + // expected-set rows and the flip were one SaveChanges, so nothing was written; nothing is + // published and the coordinator is free for the next deployment. + _log.Warning( + "DispatchDeployment {Id} abandoned: the deployment changed state while it was being dispatched " + + "(closed as a lost dispatch?) — not reopening it", msg.DeploymentId); + ResetForNext(); + return; + } } // Per-cluster mesh Phase 2: hand off to the transport router rather than publishing @@ -483,9 +504,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers /// stub stays tracked so can add to the same UPDATE. /// private static void UpdateDeploymentStatus( - OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false) + OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false, byte[]? rowVersion = null) { - var entry = TrackStub(db, id); + var entry = TrackStub(db, id, rowVersion); if (entry is null) return; entry.Entity.Status = status; entry.Property(x => x.Status).IsModified = true; @@ -507,20 +528,30 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers } /// The tracked blob-free stub for a deployment, attaching one on first use; null when the row is gone. + /// The context. + /// The deployment. + /// + /// The RowVersion the update must be conditioned on, when the caller already read one (the dispatch + /// guard); reads the current one. + /// private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry? TrackStub( - OtOpcUaConfigDbContext db, DeploymentId id) + OtOpcUaConfigDbContext db, DeploymentId id, byte[]? rowVersion = null) { var tracked = db.ChangeTracker.Entries().FirstOrDefault(e => e.Entity.DeploymentId == id.Value); if (tracked is not null) return tracked; - var stamp = db.Deployments - .AsNoTracking() - .Where(x => x.DeploymentId == id.Value) - .Select(x => new { x.RowVersion }) - .FirstOrDefault(); - if (stamp is null) return null; + if (rowVersion is null) + { + var stamp = db.Deployments + .AsNoTracking() + .Where(x => x.DeploymentId == id.Value) + .Select(x => new { x.RowVersion }) + .FirstOrDefault(); + if (stamp is null) return null; + rowVersion = stamp.RowVersion; + } return db.Deployments.Attach(new Deployment { - DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = stamp.RowVersion, + DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = rowVersion, }); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index f838a004..9a9f0742 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; @@ -111,7 +112,7 @@ public static class ServiceCollectionExtensions // coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks // inside a props factory and races the spawn it is waiting for. var meshRouter = registry.Get(); - return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: meshRouter); + return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: DeployTimings.ApplyDeadline, meshRouter: meshRouter); }, singletonOptions); @@ -128,7 +129,10 @@ public static class ServiceCollectionExtensions resolver.GetService()?["Deployment:TagConfigValidationMode"], ignoreCase: true, out var m) ? m : TagConfigValidationMode.Warn; var meshRouter = registry.Get(); - return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter); + // The SAME deadline source as the coordinator above: AdminOps closes a lost dispatch only once it + // is past the coordinator's deadline, so the two must never drift. + return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter, + applyDeadline: DeployTimings.ApplyDeadline); }, singletonOptions); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj index 507aca6c..8ee03814 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj @@ -11,6 +11,9 @@ + + + diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs index 0added3d..8c93fb1c 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -311,6 +311,77 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes }, duration: PresenceBudget); } + /// + /// Re-review 1 — the reopen race. AdminOps' lost-dispatch closure commits TimedOut AFTER the coordinator's + /// terminal guard read the row as Dispatching but BEFORE the AwaitingApplyAcks flip. The flip is + /// conditioned on the guard's RowVersion, so on SQL Server it becomes a concurrency conflict and the row + /// stays TimedOut with no expected set. Positive control: the coordinator is not wedged — the next live + /// dispatch is seeded and flipped. + /// + [Fact] + [Trait("Category", "SchemaCompliance")] + public async Task On_sql_server_a_closure_between_guard_and_flip_is_not_reopened() + { + using var sql = new SqlServerTestDatabase(); + var inner = sql.Factory(); + SeedNodes(inner, "site-a-1:4053"); + var interposer = new InterposingDbFactory(inner); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(interposer, NoTimeout)); + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart ran over an empty table + + // The race proper: a new Dispatching row whose dispatch is interleaved with an AdminOps-style closure. + var victim = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow); + interposer.OnSecondCreate(() => + { + using var db = sql.NewContext(); + db.Database.ExecuteSqlRaw( + "UPDATE Deployment SET Status = {0}, FailureReason = N'closed as lost' WHERE DeploymentId = {1}", + (int)DeploymentStatus.TimedOut, victim.Value); + }); + coordinator.Tell(new DispatchDeployment(victim, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => interposer.Fired.ShouldBeTrue(), duration: PresenceBudget); + + // Positive control + ordering: a live row dispatched after it IS flipped, so the victim's dispatch was processed. + var live = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow); + coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = sql.NewContext(); + db.Deployments.AsNoTracking().Single(d => d.DeploymentId == live.Value).Status + .ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + using var verify = sql.NewContext(); + verify.Deployments.AsNoTracking().Single(d => d.DeploymentId == victim.Value).Status + .ShouldBe(DeploymentStatus.TimedOut, "the closure must not be reopened"); + verify.NodeDeploymentStates.Count(s => s.DeploymentId == victim.Value).ShouldBe(0); + } + + /// Runs an action just before handing out the SECOND context after arming (HandleDispatch: guard, then flip). + private sealed class InterposingDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private Action? _action; + private int _countdown; + public volatile bool Fired; + + public void OnSecondCreate(Action action) + { + _action = action; + Volatile.Write(ref _countdown, 2); + } + + public OtOpcUaConfigDbContext CreateDbContext() + { + if (Volatile.Read(ref _countdown) > 0 && Interlocked.Decrement(ref _countdown) == 0) + { + _action?.Invoke(); + Fired = true; + } + return inner.CreateDbContext(); + } + } + private static DeploymentId SeedDeployment( IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) {