From b437c34f924a689177b3ab8b867f78ed6b5edb95 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:22:20 -0400 Subject: [PATCH 01/19] fix(controlplane): recover a lost dispatch instead of wedging the deploy pipeline A Deployment row left in Dispatching with no NodeDeploymentState rows (HandleDispatch threw before its SaveChanges, or the process died between the AdminOps row write and the coordinator's Tell) was recovered with an empty expected set; the deadline then hit the 0 == 0 "race" branch and returned without touching the row, so every later StartDeployment was refused AnotherDeploymentInFlight until someone hand-edited SQL. - PreStart: a recovered row with no expected-ack rows is a lost dispatch. Inside the apply deadline it is re-driven through HandleDispatch (bounded: CreatedAtUtc never moves); past it the row is closed TimedOut with a reason naming the condition. - HandleDispatch is idempotent on a re-dispatch (rows already seeded ARE the expected set). - HandleDeadline takes the race branch only when _expectedAcks.Count > 0. - The PreStart recovery read projects ArtifactBlob away (R03:3.5). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Coordinators/ConfigPublishCoordinator.cs | 99 ++++++-- .../ConfigPublishCoordinatorRecoveryTests.cs | 234 ++++++++++++++++++ 2 files changed, 316 insertions(+), 17 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index b621cc64..3c99e998 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -161,9 +161,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(DeploymentAcksTopic, Self)); using var db = _dbFactory.CreateDbContext(); + // Projected: the row carries the whole ArtifactBlob (varbinary(max)), which this recovery never + // reads — and this query runs on every singleton start and hand-over. var inflight = db.Deployments .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, d.RevisionHash, d.CreatedAtUtc }) .FirstOrDefault(); if (inflight is null) return; @@ -173,6 +176,41 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers .AsNoTracking() .ToList(); + // Resume the deadline timer using the remaining time. The deadline runs from when the + // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough + // proxy — we don't track a separate "dispatched at" column). + var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc; + var remaining = _applyDeadline - elapsed; + + if (nodeStates.Count == 0) + { + // The expected set and the AwaitingApplyAcks flip are written in ONE SaveChanges by + // HandleDispatch, so a row with no NodeDeploymentState rows is a dispatch that never completed: + // HandleDispatch threw before its SaveChanges (this actor restarted and the message was + // dropped), the process died between AdminOps' row write and the coordinator's Tell, or the + // Tell was lost across a singleton hand-over. Recovering it with an EMPTY expected set used to + // let the deadline take the "everyone acked" race branch (0 == 0) and leave the row in flight + // forever, refusing every later StartDeployment. Inside the deadline the dispatch is simply + // re-driven through the normal handler (bounded: CreatedAtUtc never moves, so a dispatch that + // keeps throwing ends up on the other branch once the deadline passes); past it the row is + // closed as TimedOut with a reason that names the condition. + if (remaining > TimeSpan.Zero) + { + _log.Warning( + "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " + + "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " + + "re-dispatching with {Remaining} of the apply deadline left", + _current, remaining); + Self.Tell(new DispatchDeployment( + _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId())); + } + else + { + Self.Tell(new DeadlineElapsed(_current.Value)); + } + return; + } + _expectedAcks = nodeStates.Select(s => NodeId.Parse(s.NodeId)).ToHashSet(NodeIdComparer); foreach (var s in nodeStates.Where(s => s.Status != NodeDeploymentStatus.Applying)) { @@ -182,11 +220,6 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId)); } - // Resume the deadline timer using the remaining time. The deadline runs from when the - // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough - // proxy — we don't track a separate "dispatched at" column). - var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc; - var remaining = _applyDeadline - elapsed; if (remaining <= TimeSpan.Zero) { Self.Tell(new DeadlineElapsed(_current.Value)); @@ -208,19 +241,34 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // Seed NodeDeploymentState rows so a failover knows which nodes were expected to ack. using (var db = _dbFactory.CreateDbContext()) { - // Discovery reuses this context rather than opening a second one — it reads the same - // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single - // context keeps the expected set and the seeded rows consistent by construction. - _expectedAcks = DiscoverDriverNodes(db); - - foreach (var node in _expectedAcks) + // Idempotent on a re-dispatch (PreStart re-drives a lost dispatch through this handler): rows + // already seeded for this deployment ARE the expected set — re-discovering would both duplicate + // the primary key and let a node toggled since the first attempt change who is expected. + var seeded = db.NodeDeploymentStates + .AsNoTracking() + .Where(x => x.DeploymentId == msg.DeploymentId.Value) + .Select(x => x.NodeId) + .ToList(); + if (seeded.Count > 0) { - db.NodeDeploymentStates.Add(new NodeDeploymentState + _expectedAcks = seeded.Select(NodeId.Parse).ToHashSet(NodeIdComparer); + } + else + { + // Discovery reuses this context rather than opening a second one — it reads the same + // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single + // context keeps the expected set and the seeded rows consistent by construction. + _expectedAcks = DiscoverDriverNodes(db); + + foreach (var node in _expectedAcks) { - NodeId = node.Value, - DeploymentId = msg.DeploymentId.Value, - Status = NodeDeploymentStatus.Applying, - }); + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = node.Value, + DeploymentId = msg.DeploymentId.Value, + Status = NodeDeploymentStatus.Applying, + }); + } } UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks); db.SaveChanges(); @@ -360,7 +408,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers msg.DeploymentId, _current); return; } - if (_acks.Count == _expectedAcks.Count) + // Only a NON-EMPTY expected set can be "fully acked": with nothing expected, 0 == 0 is not the + // everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped. + if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count) { // Race: every node acked just as the deadline fired. Already sealed/failed elsewhere. return; @@ -368,6 +418,21 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers using var db = _dbFactory.CreateDbContext(); UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut); + if (_expectedAcks.Count == 0) + { + var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); + if (d is not null) + d.FailureReason = Truncate( + "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + + "hand-over mid-dispatch) and the apply deadline has passed. Redeploy."); + db.SaveChanges(); + _log.Warning( + "Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " + + "expected and the {Deadline} apply deadline has passed. Closed as TimedOut — redeploy", + _current.Value, _applyDeadline); + ResetForNext(); + return; + } db.SaveChanges(); // Name the silent nodes, not just the counts. Since the expected set became DB-derived this // is the failure an operator meets after deploying while a configured node is switched off, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs new file mode 100644 index 00000000..86f6efa7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -0,0 +1,234 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Review 03 §2.2: a Deployment row left in Dispatching with NO NodeDeploymentState +/// rows — HandleDispatch threw before its SaveChanges, or the process died between the +/// AdminOps row write and the coordinator's Tell — used to be recovered with an EMPTY expected +/// set, at which point the deadline hit the 0 == 0 "race" branch and returned without touching +/// the row. Every later StartDeployment was then refused AnotherDeploymentInFlight forever. +/// +public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTestBase +{ + private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64)); + private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5); + + /// + /// A lost dispatch that is still inside the apply deadline is re-driven: PreStart re-dispatches it, + /// the expected set is seeded from the enabled rows, and the node's ack seals it. + /// + [Fact] + public void Recovered_row_with_no_expected_set_is_redispatched_and_then_seals() + { + var dbFactory = NewInMemoryDbFactory(); + var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + SeedNodes(dbFactory, "site-a-1:4053"); + + // No DispatchDeployment is ever sent — PreStart alone has to recover it. + var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" }); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// + /// A lost dispatch that is already past the apply deadline is closed as TimedOut with a reason + /// naming the condition — never left in Dispatching — and the next deployment proceeds + /// (positive control: it is dispatched, acked and sealed through the same coordinator). + /// + [Fact] + public void Recovered_row_past_deadline_with_no_expected_set_is_timed_out_not_wedged() + { + var dbFactory = NewInMemoryDbFactory(); + var staleId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); + SeedNodes(dbFactory, "site-a-1:4053"); + + var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + var row = db.Deployments.Single(d => d.DeploymentId == staleId.Value); + row.Status.ShouldBe(DeploymentStatus.TimedOut); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("lost"); + }, duration: PresenceBudget); + + // Positive control: the coordinator is not wedged — a fresh deployment runs end to end. + var nextId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + actor.Tell(new DispatchDeployment(nextId, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + actor.Tell(new ApplyAck(nextId, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// + /// An AwaitingApplyAcks row with no rows (the process died between the status flip and the + /// empty-fleet seal) is re-driven too, and with no enabled nodes it seals empty exactly as a fresh + /// dispatch would — the recovery keys on the missing expected set, not on the status value. + /// + [Fact] + public void Recovered_AwaitingApplyAcks_row_with_no_rows_and_no_fleet_seals_empty() + { + var dbFactory = NewInMemoryDbFactory(); + SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow); + Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + db.NodeDeploymentStates.ShouldBeEmpty(); + }, duration: PresenceBudget); + } + + /// + /// The brief's scenario end to end: HandleDispatch throws once (a transient fault seeding the + /// expected set). The coordinator restarts, recovers the Dispatching row, re-dispatches it and + /// the deployment completes; a second StartDeployment is then Accepted (positive control: a + /// normal deploy seals). + /// + [Fact] + public async Task HandleDispatch_throw_is_recovered_and_the_next_StartDeployment_succeeds() + { + var dbName = Guid.NewGuid().ToString("N"); + var adminFactory = NewInMemoryDbFactory(dbName); + var faulting = new OneShotFaultingDbFactory(NewInMemoryDbFactory(dbName)); + SeedNodes(adminFactory, "site-a-1:4053"); + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout)); + var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty())); + + // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and + // the actor would be stopped, not restarted). Identify is processed after creation, so a + // resolved selection proves PreStart completed. + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); + faulting.FailNext(); + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var first = ExpectMsg(PresenceBudget); + first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + first.DeploymentId.ShouldNotBeNull(); + + // The fault fired, the actor restarted, PreStart re-dispatched: the expected set lands. + AwaitAssert(() => + { + faulting.Faulted.ShouldBeTrue(); + using var db = adminFactory.CreateDbContext(); + db.NodeDeploymentStates.Count(s => s.DeploymentId == first.DeploymentId.Value.Value).ShouldBe(1); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + coordinator.Tell(new ApplyAck(first.DeploymentId.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = adminFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + + // Positive control: nothing is wedged — the next deploy is Accepted and seals too. + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var second = ExpectMsg(PresenceBudget); + second.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.Tell(new ApplyAck(second.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = adminFactory.CreateDbContext(); + db.Deployments.Count(d => d.Status == DeploymentStatus.Sealed).ShouldBe(2); + }, duration: PresenceBudget); + } + + private static DeploymentId SeedDeployment( + IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) + { + var id = DeploymentId.NewId(); + using var db = dbFactory.CreateDbContext(); + db.Deployments.Add(new Deployment + { + DeploymentId = id.Value, + RevisionHash = TestRevision.Value, + Status = status, + CreatedBy = "test", + CreatedAtUtc = createdAtUtc, + }); + db.SaveChanges(); + return id; + } + + private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodeIds) + { + using var db = dbFactory.CreateDbContext(); + db.ServerClusters.Add(new ServerCluster + { + ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", + NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + foreach (var nodeId in nodeIds) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0], + ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test", + }); + } + db.SaveChanges(); + } + + /// Throws from exactly one CreateDbContext call after . + private sealed class OneShotFaultingDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private int _armed; + public bool Faulted { get; private set; } + + public void FailNext() => Interlocked.Exchange(ref _armed, 1); + + public OtOpcUaConfigDbContext CreateDbContext() + { + if (Interlocked.Exchange(ref _armed, 0) == 1) + { + Faulted = true; + throw new InvalidOperationException("injected transient fault seeding the expected set"); + } + return inner.CreateDbContext(); + } + } +} From 73e74e270ddf8b9be39c010935cc1aff9c8f91fe Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:36:12 -0400 Subject: [PATCH 02/19] fix(controlplane): validate and compose a deployment from ONE snapshot read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StartDeployment read the whole config DB twice with no transaction — DraftSnapshotFactory for the validator, then ConfigComposer for the artifact, plus clusters/nodes and Script.SourceCode again in between. An AdminUI save landing between the two shipped an artifact the DraftValidator never saw. - DraftSnapshot carries ServerClusters, ClusterNodes and ApiKeys; DraftSnapshotFactory reads every table inside one SNAPSHOT transaction on a relational provider (no-op on InMemory or under an ambient transaction). - ConfigComposer.Compose(DraftSnapshot) is pure over the snapshot (ordinal-sorted keys); SnapshotAndFlattenAsync(db) delegates to it, byte-identical. - AdminOperationsActor validates topology, estimates compile cost and composes the artifact from the one snapshot it holds. - Migration EnableSnapshotIsolation: ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON (suppressTransaction; no model change). Tests: the SQL Server snapshot read ignores a row committed mid-read (positive control: a later read sees it); an edit injected between the Equipment and Tags reads is absent from the shipped artifact and rejected by the next deploy; Compose(snapshot) == SnapshotAndFlattenAsync(db). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- ...102459_EnableSnapshotIsolation.Designer.cs | 1618 +++++++++++++++++ .../20261022102459_EnableSnapshotIsolation.cs | 36 + .../Validation/DraftSnapshot.cs | 17 + .../Validation/DraftSnapshotFactory.cs | 31 + .../AdminOperations/AdminOperationsActor.cs | 29 +- .../AdminOperations/ConfigComposer.cs | 61 +- .../DraftSnapshotFactorySqlServerTests.cs | 117 ++ .../DraftSnapshotFactoryTests.cs | 30 + ...AdminOperationsActorSingleSnapshotTests.cs | 133 ++ .../ConfigComposerTests.cs | 41 + 10 files changed, 2077 insertions(+), 36 deletions(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs new file mode 100644 index 00000000..db54f3a9 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs @@ -0,0 +1,1618 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ZB.MOM.WW.OtOpcUa.Configuration; + +#nullable disable + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations +{ + [DbContext(typeof(OtOpcUaConfigDbContext))] + [Migration("20261022102459_EnableSnapshotIsolation")] + partial class EnableSnapshotIsolation + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("Microsoft.AspNetCore.DataProtection.EntityFrameworkCore.DataProtectionKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("FriendlyName") + .HasColumnType("nvarchar(max)"); + + b.Property("Xml") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("DataProtectionKeys", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ApiKey", b => + { + b.Property("ApiKeyRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ApiKeyId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedUtc") + .HasColumnType("datetime2"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("ExpiresUtc") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SecretHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.HasKey("ApiKeyRowId"); + + b.HasIndex("ApiKeyId") + .IsUnique() + .HasDatabaseName("UX_ApiKey_ApiKeyId"); + + b.HasIndex("Principal") + .HasDatabaseName("IX_ApiKey_Principal"); + + b.ToTable("ApiKey", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("AkkaPort") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValue(4053); + + b.Property("ApplicationUri") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("DashboardPort") + .HasColumnType("int"); + + b.Property("DriverConfigOverridesJson") + .HasColumnType("nvarchar(max)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("GrpcPort") + .HasColumnType("int"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("LastSeenAt") + .HasColumnType("datetime2(3)"); + + b.Property("MaintenanceMode") + .HasColumnType("bit"); + + b.Property("OpcUaPort") + .HasColumnType("int"); + + b.Property("ServiceLevelBase") + .HasColumnType("tinyint"); + + b.HasKey("NodeId"); + + b.HasIndex("ApplicationUri") + .IsUnique() + .HasDatabaseName("UX_ClusterNode_ApplicationUri"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_ClusterNode_ClusterId"); + + b.ToTable("ClusterNode", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNodeCredential", b => + { + b.Property("CredentialId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("NodeId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RotatedAt") + .HasColumnType("datetime2(3)"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.HasKey("CredentialId"); + + b.HasIndex("Kind", "Value") + .IsUnique() + .HasDatabaseName("UX_ClusterNodeCredential_Value") + .HasFilter("[Enabled] = 1"); + + b.HasIndex("NodeId", "Enabled") + .HasDatabaseName("IX_ClusterNodeCredential_NodeId"); + + b.ToTable("ClusterNodeCredential", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ConfigAuditLog", b => + { + b.Property("AuditId") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("AuditId")); + + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CorrelationId") + .HasColumnType("uniqueidentifier"); + + b.Property("DetailsJson") + .HasColumnType("nvarchar(max)"); + + b.Property("EventId") + .HasColumnType("uniqueidentifier"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("GenerationId") + .HasColumnType("bigint"); + + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Outcome") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Timestamp") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.HasKey("AuditId"); + + b.HasIndex("EventId") + .IsUnique() + .HasDatabaseName("UX_ConfigAuditLog_EventId") + .HasFilter("[EventId] IS NOT NULL"); + + b.HasIndex("GenerationId") + .HasDatabaseName("IX_ConfigAuditLog_Generation") + .HasFilter("[GenerationId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Timestamp") + .IsDescending(false, true) + .HasDatabaseName("IX_ConfigAuditLog_Cluster_Time"); + + b.ToTable("ConfigAuditLog", null, t => + { + t.HasCheckConstraint("CK_ConfigAuditLog_DetailsJson_IsJson", "DetailsJson IS NULL OR ISJSON(DetailsJson) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ConfigEdit", b => + { + b.Property("EditId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("EditedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("EditedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EntityId") + .HasColumnType("uniqueidentifier"); + + b.Property("EntityType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ExecutionId") + .HasColumnType("uniqueidentifier"); + + b.Property("FieldsJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SourceNode") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("EditId"); + + b.HasIndex("EditedAtUtc") + .HasDatabaseName("IX_ConfigEdit_EditedAt"); + + b.HasIndex("ExecutionId") + .HasDatabaseName("IX_ConfigEdit_Execution") + .HasFilter("[ExecutionId] IS NOT NULL"); + + b.HasIndex("EntityType", "EntityId") + .HasDatabaseName("IX_ConfigEdit_Entity"); + + b.ToTable("ConfigEdit", null, t => + { + t.HasCheckConstraint("CK_ConfigEdit_FieldsJson_IsJson", "ISJSON(FieldsJson) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Deployment", b => + { + b.Property("DeploymentId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ArtifactBlob") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("CreatedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("FailureReason") + .HasMaxLength(2048) + .HasColumnType("nvarchar(2048)"); + + b.Property("RevisionHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SealedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("DeploymentId"); + + b.HasIndex("CreatedAtUtc") + .HasDatabaseName("IX_Deployment_CreatedAt"); + + b.HasIndex("Status") + .HasDatabaseName("IX_Deployment_Status"); + + b.ToTable("Deployment", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Device", b => + { + b.Property("DeviceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DeviceConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DeviceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverInstanceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("DeviceRowId"); + + b.HasIndex("DeviceId") + .IsUnique() + .HasDatabaseName("UX_Device_LogicalId") + .HasFilter("[DeviceId] IS NOT NULL"); + + b.HasIndex("DriverInstanceId") + .HasDatabaseName("IX_Device_Driver"); + + b.HasIndex("DriverInstanceId", "Name") + .IsUnique() + .HasDatabaseName("UX_Device_Driver_Name"); + + b.ToTable("Device", null, t => + { + t.HasCheckConstraint("CK_Device_DeviceConfig_IsJson", "ISJSON(DeviceConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverInstance", b => + { + b.Property("DriverInstanceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DriverInstanceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ResilienceConfig") + .HasColumnType("nvarchar(max)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("DriverInstanceRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_DriverInstance_Cluster"); + + b.HasIndex("DriverInstanceId") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_LogicalId") + .HasFilter("[DriverInstanceId] IS NOT NULL"); + + b.HasIndex("RawFolderId") + .HasDatabaseName("IX_DriverInstance_RawFolder"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_ClusterRoot_Name") + .HasFilter("[RawFolderId] IS NULL"); + + b.HasIndex("ClusterId", "RawFolderId", "Name") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_Folder_Name") + .HasFilter("[RawFolderId] IS NOT NULL"); + + b.ToTable("DriverInstance", null, t => + { + t.HasCheckConstraint("CK_DriverInstance_DriverConfig_IsJson", "ISJSON(DriverConfig) = 1"); + + t.HasCheckConstraint("CK_DriverInstance_ResilienceConfig_IsJson", "ResilienceConfig IS NULL OR ISJSON(ResilienceConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Equipment", b => + { + b.Property("EquipmentRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AssetLocation") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("DeviceManualUri") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentClassRef") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EquipmentId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("EquipmentUuid") + .HasColumnType("uniqueidentifier"); + + b.Property("HardwareRevision") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("MachineCode") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Manufacturer") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ManufacturerUri") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Model") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SAPID") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SerialNumber") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SoftwareRevision") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("UnsLineId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("YearOfConstruction") + .HasColumnType("smallint"); + + b.Property("ZTag") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("EquipmentRowId"); + + b.HasIndex("EquipmentId") + .IsUnique() + .HasDatabaseName("UX_Equipment_LogicalId") + .HasFilter("[EquipmentId] IS NOT NULL"); + + b.HasIndex("EquipmentUuid") + .IsUnique() + .HasDatabaseName("UX_Equipment_Uuid"); + + b.HasIndex("MachineCode") + .HasDatabaseName("IX_Equipment_MachineCode"); + + b.HasIndex("SAPID") + .HasDatabaseName("IX_Equipment_SAPID") + .HasFilter("[SAPID] IS NOT NULL"); + + b.HasIndex("UnsLineId") + .HasDatabaseName("IX_Equipment_Line"); + + b.HasIndex("ZTag") + .HasDatabaseName("IX_Equipment_ZTag") + .HasFilter("[ZTag] IS NOT NULL"); + + b.HasIndex("UnsLineId", "Name") + .IsUnique() + .HasDatabaseName("UX_Equipment_LinePath"); + + b.ToTable("Equipment", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ExternalIdReservation", b => + { + b.Property("ReservationId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("EquipmentUuid") + .HasColumnType("uniqueidentifier"); + + b.Property("FirstPublishedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("FirstPublishedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("LastPublishedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("ReleaseReason") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("ReleasedAt") + .HasColumnType("datetime2(3)"); + + b.Property("ReleasedBy") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("ReservationId"); + + b.HasIndex("EquipmentUuid") + .HasDatabaseName("IX_ExternalIdReservation_Equipment"); + + b.HasIndex("Kind", "Value") + .IsUnique() + .HasDatabaseName("UX_ExternalIdReservation_KindValue_Active") + .HasFilter("[ReleasedAt] IS NULL"); + + b.ToTable("ExternalIdReservation", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.LdapGroupRoleMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("IsSystemWide") + .HasColumnType("bit"); + + b.Property("LdapGroup") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Role") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("Id"); + + b.HasIndex("ClusterId"); + + b.HasIndex("LdapGroup") + .HasDatabaseName("IX_LdapGroupRoleMapping_Group"); + + b.HasIndex("LdapGroup", "ClusterId") + .IsUnique() + .HasDatabaseName("UX_LdapGroupRoleMapping_Group_Cluster") + .HasFilter("[ClusterId] IS NOT NULL"); + + b.ToTable("LdapGroupRoleMapping", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeAcl", b => + { + b.Property("NodeAclRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("NodeAclId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("PermissionFlags") + .HasColumnType("int"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScopeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ScopeKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.HasKey("NodeAclRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_NodeAcl_Cluster"); + + b.HasIndex("NodeAclId") + .IsUnique() + .HasDatabaseName("UX_NodeAcl_LogicalId") + .HasFilter("[NodeAclId] IS NOT NULL"); + + b.HasIndex("Principal") + .HasDatabaseName("IX_NodeAcl_Principal"); + + b.HasIndex("ScopeKind", "ScopeId") + .HasDatabaseName("IX_NodeAcl_Scope") + .HasFilter("[ScopeId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Principal", "ScopeKind", "ScopeId") + .IsUnique() + .HasDatabaseName("UX_NodeAcl_PrincipalScope") + .HasFilter("[ScopeId] IS NOT NULL"); + + b.ToTable("NodeAcl", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeDeploymentState", b => + { + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DeploymentId") + .HasColumnType("uniqueidentifier"); + + b.Property("AppliedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("FailureReason") + .HasMaxLength(2048) + .HasColumnType("nvarchar(2048)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("StartedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("NodeId", "DeploymentId"); + + b.HasIndex("DeploymentId") + .HasDatabaseName("IX_NodeDeploymentState_Deployment"); + + b.HasIndex("Status") + .HasDatabaseName("IX_NodeDeploymentState_Status"); + + b.ToTable("NodeDeploymentState", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.PollGroup", b => + { + b.Property("PollGroupRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DriverInstanceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("IntervalMs") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PollGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("PollGroupRowId"); + + b.HasIndex("DriverInstanceId") + .HasDatabaseName("IX_PollGroup_Driver"); + + b.HasIndex("PollGroupId") + .IsUnique() + .HasDatabaseName("UX_PollGroup_LogicalId") + .HasFilter("[PollGroupId] IS NOT NULL"); + + b.ToTable("PollGroup", null, t => + { + t.HasCheckConstraint("CK_PollGroup_IntervalMs_Min", "IntervalMs >= 50"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.RawFolder", b => + { + b.Property("RawFolderRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ParentRawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.HasKey("RawFolderRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_RawFolder_Cluster"); + + b.HasIndex("RawFolderId") + .IsUnique() + .HasDatabaseName("UX_RawFolder_LogicalId") + .HasFilter("[RawFolderId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_RawFolder_ClusterRoot_Name") + .HasFilter("[ParentRawFolderId] IS NULL"); + + b.HasIndex("ClusterId", "ParentRawFolderId", "Name") + .IsUnique() + .HasDatabaseName("UX_RawFolder_Parent_Name") + .HasFilter("[ParentRawFolderId] IS NOT NULL"); + + b.ToTable("RawFolder", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Script", b => + { + b.Property("ScriptRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("Language") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SourceCode") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SourceHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("ScriptRowId"); + + b.HasIndex("ScriptId") + .IsUnique() + .HasDatabaseName("UX_Script_LogicalId") + .HasFilter("[ScriptId] IS NOT NULL"); + + b.HasIndex("SourceHash") + .HasDatabaseName("IX_Script_SourceHash"); + + b.ToTable("Script", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ScriptedAlarm", b => + { + b.Property("ScriptedAlarmRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AlarmType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("HistorizeToAveva") + .HasColumnType("bit"); + + b.Property("MessageTemplate") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PredicateScriptId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Retain") + .HasColumnType("bit"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptedAlarmId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Severity") + .HasColumnType("int"); + + b.HasKey("ScriptedAlarmRowId"); + + b.HasIndex("PredicateScriptId") + .HasDatabaseName("IX_ScriptedAlarm_Script"); + + b.HasIndex("ScriptedAlarmId") + .IsUnique() + .HasDatabaseName("UX_ScriptedAlarm_LogicalId") + .HasFilter("[ScriptedAlarmId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "Name") + .IsUnique() + .HasDatabaseName("UX_ScriptedAlarm_EquipmentPath"); + + b.ToTable("ScriptedAlarm", null, t => + { + t.HasCheckConstraint("CK_ScriptedAlarm_AlarmType", "AlarmType IN ('AlarmCondition','LimitAlarm','OffNormalAlarm','DiscreteAlarm')"); + + t.HasCheckConstraint("CK_ScriptedAlarm_Severity_Range", "Severity BETWEEN 1 AND 1000"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", b => + { + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Enterprise") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2(3)"); + + b.Property("ModifiedBy") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("NodeCount") + .HasColumnType("tinyint"); + + b.Property("Notes") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("RedundancyMode") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Site") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("ClusterId"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("UX_ServerCluster_Name"); + + b.HasIndex("Site") + .HasDatabaseName("IX_ServerCluster_Site"); + + b.ToTable("ServerCluster", null, t => + { + t.HasCheckConstraint("CK_ServerCluster_RedundancyMode_NodeCount", "((NodeCount = 1 AND RedundancyMode = 'None') OR (NodeCount = 2 AND RedundancyMode IN ('Warm', 'Hot')))"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Tag", b => + { + b.Property("TagRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AccessLevel") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("DataType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("DeviceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PollGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("TagConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("TagId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("WriteIdempotent") + .HasColumnType("bit"); + + b.HasKey("TagRowId"); + + b.HasIndex("TagId") + .IsUnique() + .HasDatabaseName("UX_Tag_LogicalId") + .HasFilter("[TagId] IS NOT NULL"); + + b.HasIndex("DeviceId", "Name") + .IsUnique() + .HasDatabaseName("UX_Tag_Device_Name") + .HasFilter("[TagGroupId] IS NULL"); + + b.HasIndex("DeviceId", "TagGroupId") + .HasDatabaseName("IX_Tag_Device"); + + b.HasIndex("DeviceId", "TagGroupId", "Name") + .IsUnique() + .HasDatabaseName("UX_Tag_Group_Name") + .HasFilter("[TagGroupId] IS NOT NULL"); + + b.ToTable("Tag", null, t => + { + t.HasCheckConstraint("CK_Tag_TagConfig_IsJson", "ISJSON(TagConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.TagGroup", b => + { + b.Property("TagGroupRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DeviceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ParentTagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.Property("TagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("TagGroupRowId"); + + b.HasIndex("DeviceId") + .HasDatabaseName("IX_TagGroup_Device"); + + b.HasIndex("TagGroupId") + .IsUnique() + .HasDatabaseName("UX_TagGroup_LogicalId") + .HasFilter("[TagGroupId] IS NOT NULL"); + + b.HasIndex("DeviceId", "Name") + .IsUnique() + .HasDatabaseName("UX_TagGroup_DeviceRoot_Name") + .HasFilter("[ParentTagGroupId] IS NULL"); + + b.HasIndex("DeviceId", "ParentTagGroupId", "Name") + .IsUnique() + .HasDatabaseName("UX_TagGroup_Parent_Name") + .HasFilter("[ParentTagGroupId] IS NOT NULL"); + + b.ToTable("TagGroup", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsArea", b => + { + b.Property("UnsAreaRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("UnsAreaId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsAreaRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_UnsArea_Cluster"); + + b.HasIndex("UnsAreaId") + .IsUnique() + .HasDatabaseName("UX_UnsArea_LogicalId") + .HasFilter("[UnsAreaId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_UnsArea_ClusterName"); + + b.ToTable("UnsArea", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsLine", b => + { + b.Property("UnsLineRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("UnsAreaId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("UnsLineId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsLineRowId"); + + b.HasIndex("UnsAreaId") + .HasDatabaseName("IX_UnsLine_Area"); + + b.HasIndex("UnsLineId") + .IsUnique() + .HasDatabaseName("UX_UnsLine_LogicalId") + .HasFilter("[UnsLineId] IS NOT NULL"); + + b.HasIndex("UnsAreaId", "Name") + .IsUnique() + .HasDatabaseName("UX_UnsLine_AreaName"); + + b.ToTable("UnsLine", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsTagReference", b => + { + b.Property("UnsTagReferenceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DisplayNameOverride") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.Property("TagId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("UnsTagReferenceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsTagReferenceRowId"); + + b.HasIndex("EquipmentId") + .HasDatabaseName("IX_UnsTagReference_Equipment"); + + b.HasIndex("TagId") + .HasDatabaseName("IX_UnsTagReference_Tag"); + + b.HasIndex("UnsTagReferenceId") + .IsUnique() + .HasDatabaseName("UX_UnsTagReference_LogicalId") + .HasFilter("[UnsTagReferenceId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "TagId") + .IsUnique() + .HasDatabaseName("UX_UnsTagReference_Equip_Tag"); + + b.ToTable("UnsTagReference", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.VirtualTag", b => + { + b.Property("VirtualTagRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ChangeTriggered") + .HasColumnType("bit"); + + b.Property("DataType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Historize") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("TimerIntervalMs") + .HasColumnType("int"); + + b.Property("VirtualTagId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("VirtualTagRowId"); + + b.HasIndex("ScriptId") + .HasDatabaseName("IX_VirtualTag_Script"); + + b.HasIndex("VirtualTagId") + .IsUnique() + .HasDatabaseName("UX_VirtualTag_LogicalId") + .HasFilter("[VirtualTagId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "Name") + .IsUnique() + .HasDatabaseName("UX_VirtualTag_EquipmentPath"); + + b.ToTable("VirtualTag", null, t => + { + t.HasCheckConstraint("CK_VirtualTag_TimerInterval_Min", "TimerIntervalMs IS NULL OR TimerIntervalMs >= 50"); + + t.HasCheckConstraint("CK_VirtualTag_Trigger_AtLeastOne", "ChangeTriggered = 1 OR TimerIntervalMs IS NOT NULL"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany("Nodes") + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNodeCredential", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", "Node") + .WithMany("Credentials") + .HasForeignKey("NodeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Node"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverInstance", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.LdapGroupRoleMapping", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeDeploymentState", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Deployment", "Deployment") + .WithMany() + .HasForeignKey("DeploymentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", "Node") + .WithMany() + .HasForeignKey("NodeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Deployment"); + + b.Navigation("Node"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.RawFolder", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", null) + .WithMany("RawFolders") + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsArea", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.Navigation("Credentials"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", b => + { + b.Navigation("Nodes"); + + b.Navigation("RawFolders"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs new file mode 100644 index 00000000..58cf9a7e --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs @@ -0,0 +1,36 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations +{ + /// + /// Enables ALLOW_SNAPSHOT_ISOLATION on the config database so the deploy path can read every + /// table inside one SNAPSHOT transaction (DraftSnapshotFactory.FromConfigDbAsync): + /// the validator and the artifact composer then see one consistent state instead of two reads + /// an AdminUI save can land between. No schema change — the model snapshot is untouched. + /// + /// + /// ALTER DATABASE cannot run inside the user transaction the migrator wraps each migration in, + /// hence suppressTransaction: true. Turning the option ON does not need exclusive access (unlike + /// READ_COMMITTED_SNAPSHOT); it waits for in-flight transactions and then takes effect. Down + /// leaves it ON: a database that has run with it is not harmed by keeping it, and switching it OFF + /// would fail any deploy running at that instant. + /// + public partial class EnableSnapshotIsolation : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + "ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON;", + suppressTransaction: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + // Deliberately a no-op; see the class remarks. + } + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs index 0c0c47d9..fd7dc35a 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs @@ -66,6 +66,23 @@ public sealed class DraftSnapshot /// silently dropped when the permission trie is built, so the grant has to be caught here instead. public IReadOnlyList NodeAcls { get; init; } = []; + /// + /// Cluster rows. Not read by the row-level rules; carried so the deploy path validates the cluster + /// topology () and composes the artifact from + /// the SAME read as every other table — a second read between validate and compose is exactly how + /// an edit ships unvalidated. + /// + public IReadOnlyList ServerClusters { get; init; } = []; + + /// Cluster-node rows; see for why they ride the snapshot. + public IReadOnlyList ClusterNodes { get; init; } = []; + + /// + /// Data-plane API keys (peppered HMAC only, never a secret). Not validated; carried so the artifact + /// composer ships the keys a driver-only node authenticates against from the one consistent read. + /// + public IReadOnlyList ApiKeys { get; init; } = []; + /// Prior Equipment rows (any generation, same cluster) for stability checks. public IReadOnlyList PriorEquipment { get; init; } = []; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs index 3bfd0a5c..154d664c 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs @@ -1,3 +1,4 @@ +using System.Data; using Microsoft.EntityFrameworkCore; namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation; @@ -21,6 +22,16 @@ namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation; /// / are left null so the path-length rule uses its /// conservative upper bound. /// +/// +/// One consistent read. On a relational provider the tables are read inside a +/// transaction, so a /uns or /raw save that +/// lands mid-read cannot leave half the tables on the new state and half on the old. The deploy path +/// validates AND composes its artifact from this one snapshot, which is what makes "the artifact the +/// validator never saw" unrepresentable. SQL Server requires ALLOW_SNAPSHOT_ISOLATION ON +/// (migration EnableSnapshotIsolation); without it the first query fails with error 3952 and +/// the deploy is rejected rather than silently read unisolated. A non-relational provider (the +/// InMemory test provider) and a context already inside a transaction read without one. +/// /// public static class DraftSnapshotFactory { @@ -29,6 +40,23 @@ public static class DraftSnapshotFactory /// Cancellation token. /// A snapshot populated from the live DB, ready for . public static async Task FromConfigDbAsync(OtOpcUaConfigDbContext db, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(db); + + var ownsTransaction = db.Database.IsRelational() && db.Database.CurrentTransaction is null; + var tx = ownsTransaction + ? await db.Database.BeginTransactionAsync(IsolationLevel.Snapshot, ct).ConfigureAwait(false) + : null; + await using (tx) + { + var snapshot = await ReadAsync(db, ct).ConfigureAwait(false); + // Read-only: committing releases the snapshot's version-store reservation; nothing was written. + if (tx is not null) await tx.CommitAsync(ct).ConfigureAwait(false); + return snapshot; + } + } + + private static async Task ReadAsync(OtOpcUaConfigDbContext db, CancellationToken ct) => new DraftSnapshot { // GenerationId=0 and null Enterprise/Site are intentional conservative fallbacks at the @@ -36,6 +64,8 @@ public static class DraftSnapshotFactory // EquipmentUuid checks run in separate validator passes — no rule here reads these fields. GenerationId = 0, // generation model dropped; placeholder (no rule reads it) ClusterId = string.Empty, // global snapshot; rules compare entity ClusterId fields, not this + ServerClusters = await db.ServerClusters.AsNoTracking().ToListAsync(ct), + ClusterNodes = await db.ClusterNodes.AsNoTracking().ToListAsync(ct), // v3: Namespace entity retired — no namespace list to materialize. The Raw tree // (RawFolders/TagGroups) + UnsTagReferences feed the v3 rules (charset, tagname length, // effective-leaf uniqueness). @@ -53,6 +83,7 @@ public static class DraftSnapshotFactory Scripts = await db.Scripts.AsNoTracking().ToListAsync(ct), PollGroups = await db.PollGroups.AsNoTracking().ToListAsync(ct), NodeAcls = await db.NodeAcls.AsNoTracking().ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().ToListAsync(ct), PriorEquipment = [], // intentional: no prior-generation table to diff against at this level ActiveReservations = await db.ExternalIdReservations .AsNoTracking() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index d569fd08..4aabe3db 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -223,18 +223,14 @@ public sealed class AdminOperationsActor : ReceiveActor // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at // Hot/Warm and the constraint stays green, which would boot the runtime into an - // InvalidTopology band. ValidateClusterTopology catches that drift, but it isn't carried on - // the generation-versioned DraftSnapshot (the cluster/node rows aren't versioned), so it must - // be run separately here against the live rows. Read-only (AsNoTracking); errors fold into the - // same reject summary alongside the snapshot rules so a deploy failing either check is - // rejected with both sets of messages. ClusterId-ordered for a deterministic summary. - var clusters = await db.ServerClusters - .AsNoTracking() - .OrderBy(c => c.ClusterId) - .ToListAsync(); - var nodesByCluster = (await db.ClusterNodes - .AsNoTracking() - .ToListAsync()) + // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of + // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows. + // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing + // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary. + // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT + // transaction), not a second query that could observe a later edit. + var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList(); + var nodesByCluster = draft.ClusterNodes .GroupBy(n => n.ClusterId, StringComparer.Ordinal) .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); foreach (var cluster in clusters) @@ -310,8 +306,9 @@ public sealed class AdminOperationsActor : ReceiveActor try { const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) - var scriptSources = await db.Scripts.AsNoTracking().Select(s => s.SourceCode).ToListAsync(); - var compiled = scriptSources + // Projected from the snapshot already in hand — Script.SourceCode used to be read three times. + var compiled = draft.Scripts + .Select(s => s.SourceCode) .Where(src => !PassthroughScript.TryMatch(src, out _)) .Distinct(StringComparer.Ordinal) .Count(); @@ -331,7 +328,9 @@ public sealed class AdminOperationsActor : ReceiveActor advisory = null; } - var artifact = await ConfigComposer.SnapshotAndFlattenAsync(db); + // Composed from the validated snapshot, never from a second read: an edit that lands between + // validation and here is simply not in this artifact (R03:3.2). + var artifact = ConfigComposer.Compose(draft); var deploymentId = DeploymentId.NewId(); var revHash = RevisionHash.Parse(artifact.RevisionHash); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs index d73dd7b3..6f332143 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs @@ -1,9 +1,8 @@ using System.IO.Compression; using System.Security.Cryptography; using System.Text.Json; -using Microsoft.EntityFrameworkCore; using ZB.MOM.WW.OtOpcUa.Configuration; -using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; @@ -25,45 +24,65 @@ public static class ConfigComposer }; /// Reads the current configuration and returns a deterministic snapshot blob with revision hash. + /// + /// One consistent read (, a SNAPSHOT transaction on + /// SQL Server) handed to . The deploy path does not call this: it validates and + /// composes from the one it already holds, so nothing can land between the + /// validator's read and the composer's. This overload serves the drift probe and tests. + /// /// The configuration database context. /// The cancellation token for the operation. /// A deterministic containing the serialised snapshot bytes and its revision hash. public static async Task SnapshotAndFlattenAsync( OtOpcUaConfigDbContext db, CancellationToken ct = default) + => Compose(await DraftSnapshotFactory.FromConfigDbAsync(db, ct).ConfigureAwait(false)); + + /// + /// Serialises an already-read into the deterministic artifact. Pure over + /// its input: the same snapshot always yields the same bytes and hash, whatever the DB holds by now. + /// + /// The rows to compose — the SAME rows the validator was run over. + /// The compressed artifact blob and its revision hash. + public static ConfigArtifact Compose(DraftSnapshot snapshot) { + ArgumentNullException.ThrowIfNull(snapshot); + // v3 greenfield snapshot: the Raw tree (RawFolders → DriverInstances(+RawFolderId) → Devices(+DeviceConfig) // → TagGroups → Tags(DeviceId/TagGroupId, no EquipmentId/FolderPath/DriverInstanceId)) plus the UNS // projection (UnsAreas/UnsLines/Equipment/UnsTagReferences) and the retained VirtualTags/ScriptedAlarms/ // PollGroups/NodeAcls/Scripts. The Namespaces snapshot is RETIRED (the two OPC UA namespaces are implicit). // RevisionHash inputs follow the blob automatically — adding/removing a table shifts the SHA-256. - var snapshot = new + // Every collection is sorted by its natural key with an ORDINAL comparer so the hash is independent of + // both EF row order and the process culture (a culture-aware sort could order "a-1"/"A-1" differently + // on two central nodes and make the same config hash twice). + var ordered = new { - Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), - Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), - RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), - DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), - Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), - TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), - Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), - Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), - UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), - PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), - UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), - UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), - NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + Clusters = snapshot.ServerClusters.OrderBy(x => x.ClusterId, StringComparer.Ordinal).ToList(), + Nodes = snapshot.ClusterNodes.OrderBy(x => x.NodeId, StringComparer.Ordinal).ToList(), + RawFolders = snapshot.RawFolders.OrderBy(x => x.RawFolderId, StringComparer.Ordinal).ToList(), + DriverInstances = snapshot.DriverInstances.OrderBy(x => x.DriverInstanceId, StringComparer.Ordinal).ToList(), + Devices = snapshot.Devices.OrderBy(x => x.DeviceId, StringComparer.Ordinal).ToList(), + TagGroups = snapshot.TagGroups.OrderBy(x => x.TagGroupId, StringComparer.Ordinal).ToList(), + Equipment = snapshot.Equipment.OrderBy(x => x.EquipmentId, StringComparer.Ordinal).ToList(), + Tags = snapshot.Tags.OrderBy(x => x.TagId, StringComparer.Ordinal).ToList(), + UnsTagReferences = snapshot.UnsTagReferences.OrderBy(x => x.UnsTagReferenceId, StringComparer.Ordinal).ToList(), + PollGroups = snapshot.PollGroups.OrderBy(x => x.PollGroupId, StringComparer.Ordinal).ToList(), + UnsAreas = snapshot.UnsAreas.OrderBy(x => x.UnsAreaId, StringComparer.Ordinal).ToList(), + UnsLines = snapshot.UnsLines.OrderBy(x => x.UnsLineId, StringComparer.Ordinal).ToList(), + NodeAcls = snapshot.NodeAcls.OrderBy(x => x.NodeAclId, StringComparer.Ordinal).ToList(), // the data-plane credentials travel with the config, because a driver-only node // has no ConfigDb to read them from (per-cluster mesh Phase 4) and would otherwise deny every // UserName token. Only the peppered HMAC of each secret ships — never the secret — and the // pepper is node configuration that is deliberately NOT in the artifact, so a stolen blob // still cannot be turned into a working credential. This is also why revoking a key reaches a // driver-only node only on the next deployment; see the ApiKey entity's remarks. - ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), - Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), - VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), - ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + ApiKeys = snapshot.ApiKeys.OrderBy(x => x.ApiKeyId, StringComparer.Ordinal).ToList(), + Scripts = snapshot.Scripts.OrderBy(x => x.ScriptId, StringComparer.Ordinal).ToList(), + VirtualTags = snapshot.VirtualTags.OrderBy(x => x.VirtualTagId, StringComparer.Ordinal).ToList(), + ScriptedAlarms = snapshot.ScriptedAlarms.OrderBy(x => x.ScriptedAlarmId, StringComparer.Ordinal).ToList(), }; - return Pack(JsonSerializer.SerializeToUtf8Bytes(snapshot, JsonOptions)); + return Pack(JsonSerializer.SerializeToUtf8Bytes(ordered, JsonOptions)); } /// diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs new file mode 100644 index 00000000..aa16ead6 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs @@ -0,0 +1,117 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Tests; + +/// +/// R03:3.2 on the real provider: the migrated config database allows SNAPSHOT isolation (migration +/// EnableSnapshotIsolation) and reads under +/// it. The InMemory tests cannot see either — the provider has no transactions — so this is the one +/// place the deploy path's "one consistent read" is proven against SQL Server. +/// +[Trait("Category", "SchemaCompliance")] +[Collection(nameof(SchemaComplianceCollection))] +public sealed class DraftSnapshotFactorySqlServerTests +{ + private readonly SchemaComplianceFixture _fixture; + + /// Initializes a new instance of the class. + /// The migrated test database. + public DraftSnapshotFactorySqlServerTests(SchemaComplianceFixture fixture) => _fixture = fixture; + + /// The migration leaves the database with snapshot isolation allowed. + [Fact] + public void Migrated_database_allows_snapshot_isolation() + { + using var conn = _fixture.OpenConnection(); + using var cmd = new SqlCommand( + "SELECT snapshot_isolation_state_desc FROM sys.databases WHERE name = DB_NAME();", conn); + cmd.ExecuteScalar().ShouldBe("ON"); + } + + /// + /// A row committed by another connection while the snapshot read is in progress is NOT observed by + /// the tables read after it — every table reflects the instant the transaction started. Positive + /// control: a read started after the commit does see the row. + /// + [Fact] + public async Task FromConfigDb_reads_every_table_as_of_one_instant() + { + var clusterId = $"SNAP-{Guid.NewGuid():N}"[..24]; + var scriptId = $"late-{Guid.NewGuid():N}"; + await using (var seed = NewContext()) + { + seed.ServerClusters.Add(new ServerCluster + { + ClusterId = clusterId, Name = "snapshot-read", Enterprise = "zb", Site = "test", + NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + await seed.SaveChangesAsync(); + } + + try + { + var options = new DbContextOptionsBuilder() + .UseSqlServer(_fixture.ConnectionString) + .AddInterceptors(new LateEditInterceptor(_fixture.ConnectionString, clusterId, scriptId)) + .Options; + + await using (var db = new OtOpcUaConfigDbContext(options)) + { + var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(db); + // The interceptor committed a Script while ServerClusters (the FIRST table read) was being + // materialised. Scripts is read later in the same transaction and must not see it. + snapshot.ServerClusters.ShouldContain(c => c.ClusterId == clusterId); + snapshot.Scripts.ShouldNotContain(s => s.ScriptId == scriptId); + } + + await using (var verify = NewContext()) + { + // Positive control: the edit really landed, and a read started afterwards sees it. + (await verify.Scripts.AnyAsync(s => s.ScriptId == scriptId)).ShouldBeTrue(); + var later = await DraftSnapshotFactory.FromConfigDbAsync(verify); + later.Scripts.ShouldContain(s => s.ScriptId == scriptId); + } + } + finally + { + await using var cleanup = NewContext(); + await cleanup.Scripts.Where(s => s.ScriptId == scriptId).ExecuteDeleteAsync(); + await cleanup.ServerClusters.Where(c => c.ClusterId == clusterId).ExecuteDeleteAsync(); + } + } + + private OtOpcUaConfigDbContext NewContext() => new( + new DbContextOptionsBuilder().UseSqlServer(_fixture.ConnectionString).Options); + + /// + /// Commits a Script on a SEPARATE connection the first time the snapshot read materialises the + /// seeded ServerCluster row — i.e. mid-read, after the first table and before Scripts. + /// + private sealed class LateEditInterceptor(string connectionString, string clusterId, string scriptId) + : IMaterializationInterceptor + { + private int _fired; + + public object InitializedInstance(MaterializationInterceptionData materializationData, object entity) + { + if (entity is ServerCluster c && c.ClusterId == clusterId && Interlocked.Exchange(ref _fired, 1) == 0) + { + using var other = new OtOpcUaConfigDbContext( + new DbContextOptionsBuilder().UseSqlServer(connectionString).Options); + other.Scripts.Add(new Script + { + ScriptId = scriptId, Name = "late edit", SourceCode = "return 1;", SourceHash = "late-hash", + }); + other.SaveChanges(); + } + return entity; + } + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs index 9ad5806a..df1fad5a 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs @@ -103,6 +103,36 @@ public sealed class DraftSnapshotFactoryTests : IDisposable snapshot.ActiveReservations[0].ReleasedAt.ShouldBeNull(); } + /// + /// R03:3.2 — the deploy path validates AND composes from ONE snapshot, so the tables the composer + /// needs beyond the validator's set (clusters, nodes, API keys) must ride the same read. + /// + [Fact] + public async Task FromConfigDb_carries_clusters_nodes_and_api_keys_for_the_composer() + { + _db.ServerClusters.Add(new ServerCluster + { + ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", + NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + _db.ClusterNodes.Add(new ClusterNode + { + NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1", + ApplicationUri = "urn:OtOpcUa:site-a-1", Enabled = true, CreatedBy = "test", + }); + _db.ApiKeys.Add(new ApiKey + { + ApiKeyId = "k1", Name = "scada", SecretHash = "hash", Principal = "scadabridge", Enabled = true, + }); + await _db.SaveChangesAsync(); + + var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(_db); + + snapshot.ServerClusters.ShouldHaveSingleItem().ClusterId.ShouldBe("SITE-A"); + snapshot.ClusterNodes.ShouldHaveSingleItem().NodeId.ShouldBe("site-a-1:4053"); + snapshot.ApiKeys.ShouldHaveSingleItem().ApiKeyId.ShouldBe("k1"); + } + private void SeedEquipment(string equipmentId) { var uuid = Guid.NewGuid(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs new file mode 100644 index 00000000..aa5ded29 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs @@ -0,0 +1,133 @@ +using System.IO.Compression; +using System.Text; +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Storage; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:3.2 — StartDeployment used to read the whole config DB twice: once for the validator, +/// once for the composer. An edit landing between the two shipped an artifact the validator never saw. +/// The deploy now validates and composes ONE snapshot, so an edit that lands mid-deploy is either +/// validated (it was in the snapshot) or absent from the artifact (it was not) — never shipped unseen. +/// +public sealed class AdminOperationsActorSingleSnapshotTests : ControlPlaneActorTestBase +{ + private const string LateEquipmentId = "EQ-late-and-NOT-canonical"; + + /// + /// An Equipment with a non-canonical id (the validator rejects those with EquipmentIdNotDerived) + /// is committed by another context while the snapshot read is in progress, AFTER the Equipment table + /// has been read. The validator cannot see it, so the deploy is Accepted — and the artifact must not + /// carry it either. Positive control: the row really landed in the DB, and a deploy started after it + /// is Rejected on exactly that rule. + /// + [Fact] + public void Edit_landing_between_validate_and_compose_cannot_ship_unvalidated() + { + // One shared root: an InMemory store is scoped to the internal service provider, and a context + // carrying an interceptor builds its own provider — without the shared root the intercepting + // context would read an EMPTY database and the whole test would be vacuous. + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var plain = new RootedDbFactory(dbName, root, interceptor: null); + using (var db = plain.CreateDbContext()) + { + // A Tag to materialise — Tags is read AFTER Equipment by the snapshot factory, so an insert + // fired on the first Tag lands "between" the two tables of one read. + db.DriverInstances.Add(new DriverInstance + { + DriverInstanceId = "d", ClusterId = "", Name = "drv", DriverType = "Modbus", DriverConfig = "{}", + }); + db.Devices.Add(new Device { DeviceId = "dev", DriverInstanceId = "d", Name = "dev", DeviceConfig = "{}" }); + db.Tags.Add(new Tag + { + TagId = "tag-speed", DeviceId = "dev", Name = "speed", DataType = "Float", + AccessLevel = TagAccessLevel.Read, TagConfig = "{}", + }); + db.SaveChanges(); + } + + var injecting = new RootedDbFactory(dbName, root, new LateEquipmentInterceptor(plain)); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(injecting, coordinator.Ref, Enumerable.Empty())); + + actor.Tell(new StartDeployment("joe", CorrelationId.NewId())); + + var reply = ExpectMsg(PresenceBudget); + reply.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, reply.Message); + coordinator.ExpectMsg(PresenceBudget); + + using (var verify = plain.CreateDbContext()) + { + // The injection happened... + verify.Equipment.Any(e => e.EquipmentId == LateEquipmentId).ShouldBeTrue(); + // ...and the shipped artifact does not contain what the validator never saw. + var row = verify.Deployments.Single(); + Inflate(row.ArtifactBlob).ShouldNotContain(LateEquipmentId); + // The coordinator is a probe here, so seal the row by hand to let the next deploy through. + row.Status = DeploymentStatus.Sealed; + verify.SaveChanges(); + } + + // Positive control: now that the row is visible to a fresh read, the gate rejects it. + actor.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var second = ExpectMsg(PresenceBudget); + second.Outcome.ShouldBe(StartDeploymentOutcome.Rejected); + second.Message.ShouldNotBeNull(); + second.Message.ShouldContain("EquipmentIdNotDerived"); + } + + private static string Inflate(byte[] blob) + { + using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress); + using var reader = new StreamReader(gz, Encoding.UTF8); + return reader.ReadToEnd(); + } + + /// On the first Tag materialised, commits a non-canonical Equipment through ANOTHER context. + private sealed class LateEquipmentInterceptor(IDbContextFactory other) + : IMaterializationInterceptor + { + private int _fired; + + public object InitializedInstance(MaterializationInterceptionData materializationData, object entity) + { + if (entity is Tag && Interlocked.Exchange(ref _fired, 1) == 0) + { + using var db = other.CreateDbContext(); + db.Equipment.Add(new Equipment + { + EquipmentUuid = Guid.NewGuid(), EquipmentId = LateEquipmentId, Name = "late", + UnsLineId = "line-a", MachineCode = "m", + }); + db.SaveChanges(); + } + return entity; + } + } + + private sealed class RootedDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor? interceptor) + : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() + { + var builder = new DbContextOptionsBuilder().UseInMemoryDatabase(dbName, root); + if (interceptor is not null) builder.AddInterceptors(interceptor); + return new OtOpcUaConfigDbContext(builder.Options); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs index 4e431151..674ececd 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs @@ -222,6 +222,47 @@ public sealed class ConfigComposerTests : ControlPlaneActorTestBase withoutOverride.ResilienceConfig.ShouldBeNull("an unset override stays null ⇒ tier defaults at spawn"); } + /// + /// R03:3.2 — the deploy path composes from the it + /// validated rather than re-reading the DB. over the factory's + /// snapshot must therefore be byte-identical to the DB overload, or the drift probe (which still + /// uses the DB overload) would report every deployment as drifted. + /// + [Fact] + public async Task Compose_over_the_factory_snapshot_is_byte_identical_to_the_db_overload() + { + var f = NewInMemoryDbFactory(); + await using (var db = f.CreateDbContext()) + { + db.ServerClusters.Add(NewCluster("cluster-b")); + db.ServerClusters.Add(NewCluster("cluster-a")); + db.ApiKeys.Add(new ApiKey + { + ApiKeyId = "k1", Name = "scada", SecretHash = "h", Principal = "scadabridge", CreatedUtc = FixedTimestamp, + }); + db.Scripts.Add(new Script { ScriptId = "s-1", Name = "s", SourceCode = "return 1;", SourceHash = "x" }); + await db.SaveChangesAsync(); + } + + var viaDb = await ConfigComposer.SnapshotAndFlattenAsync(f.CreateDbContext()); + var snapshot = await Configuration.Validation.DraftSnapshotFactory.FromConfigDbAsync(f.CreateDbContext()); + var viaSnapshot = ConfigComposer.Compose(snapshot); + + viaSnapshot.RevisionHash.ShouldBe(viaDb.RevisionHash); + viaSnapshot.Blob.ShouldBe(viaDb.Blob); + // The composed content is the snapshot's, not something re-read: the keys and clusters are in it. + var json = Inflate(viaSnapshot.Blob); + json.ShouldContain("\"ApiKeyId\":\"k1\""); + json.IndexOf("cluster-a", StringComparison.Ordinal).ShouldBeLessThan(json.IndexOf("cluster-b", StringComparison.Ordinal)); + } + + private static string Inflate(byte[] blob) + { + using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress); + using var reader = new StreamReader(gz, Encoding.UTF8); + return reader.ReadToEnd(); + } + private static readonly DateTime FixedTimestamp = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); private static ServerCluster NewCluster(string id) => new() From ae84253773481894b413052f3efcb1ad923f4e90 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:41:35 -0400 Subject: [PATCH 03/19] fix(controlplane): keep the admin singleton's mailbox free during Test Connect and deploy compose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReceiveAsync suspended the mailbox for the whole of every handler, so one Test Connect against a black-holed PLC (60 s clamp) or one large deploy compose stalled every other admin operation fleet-wide — AdminUI alarm acks/shelves, driver restarts, the next deploy. - TestDriverConnect runs the probe on the thread pool and pipes a TestDriverConnectResult straight to the requester (every path yields a result, never Status.Failure). - StartDeployment serialises only the in-flight check (DB row + an actor-local compose-in-flight flag, because the Deployment row is written at the END of the compose); the snapshot read, validation, gzip and row write run off-actor and return as DeployComposed / DeployComposeFailed, where the coordinator is told and the requester answered. Tests: an alarm ack completes in < 1 s while a 60 s probe is pending and while a compose is parked on a gated DbContext (positive controls: both still complete afterwards); a second StartDeployment during an in-flight compose is refused. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdminOperations/AdminOperationsActor.cs | 454 +++++++++++------- .../AdminOperationsActorMailboxTests.cs | 159 ++++++ 2 files changed, 432 insertions(+), 181 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index 4aabe3db..8b1c9ddb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -36,6 +36,9 @@ public sealed class AdminOperationsActor : ReceiveActor private readonly ILoggingAdapter _log = Context.GetLogger(); + /// The StartDeployment whose compose is running off-actor, or null. See . + private StartDeployment? _composeInFlight; + /// Creates actor props for the admin operations actor. /// Factory for creating config database contexts. /// Reference to the deployment coordinator actor. @@ -74,8 +77,12 @@ public sealed class AdminOperationsActor : ReceiveActor _tagConfigValidationMode = tagConfigValidationMode; _meshRouter = meshRouter; - ReceiveAsync(HandleStartDeploymentAsync); - ReceiveAsync(HandleTestDriverConnectAsync); + // Synchronous receives: the compose and the probe run off-actor and come back as messages, so + // the mailbox is never suspended for the length of a DB snapshot or a PLC connect timeout. + Receive(HandleStartDeployment); + Receive(HandleDeployComposed); + Receive(HandleDeployComposeFailed); + Receive(HandleTestDriverConnect); ReceiveAsync(HandleRestartDriverAsync); ReceiveAsync(HandleReconnectDriverAsync); Receive(HandleAcknowledgeAlarm); @@ -181,19 +188,42 @@ public sealed class AdminOperationsActor : ReceiveActor } } - private async Task HandleStartDeploymentAsync(StartDeployment msg) + /// + /// Serialised part of StartDeployment: the in-flight check. Everything after it — the + /// snapshot read, validation, gzip and the row write — runs on the thread pool and comes back as a + /// / message, so a large compose no + /// longer suspends the mailbox (R03:3.1) and alarm acks keep flowing while it runs. + /// + /// + /// is what keeps the check correct once the compose is off-actor: the + /// Deployment row is written at the END of the compose, so a second request arriving mid-compose + /// would pass a DB-only check and two artifacts would race for one coordinator. The flag is cleared + /// when the outcome message lands, on both branches. + /// + private void HandleStartDeployment(StartDeployment msg) { var replyTo = Sender; + if (_composeInFlight is { } inFlightRequest) + { + replyTo.Tell(new StartDeploymentResult( + StartDeploymentOutcome.AnotherDeploymentInFlight, + DeploymentId: null, + RevisionHash: null, + Message: $"A deployment requested by {inFlightRequest.CreatedBy} is still being composed.", + msg.CorrelationId)); + return; + } + try { - await using var db = await _dbFactory.CreateDbContextAsync(); - // Refuse if any deployment is already in flight — keeps the coordinator's state - // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. - var inflight = await db.Deployments + // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. One indexed + // projection (IX_Deployment_Status), synchronous on purpose: it IS the serialised part. + using var db = _dbFactory.CreateDbContext(); + var inflight = db.Deployments .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) .Select(d => d.DeploymentId) - .FirstOrDefaultAsync(); + .FirstOrDefault(); if (inflight != Guid.Empty) { replyTo.Tell(new StartDeploymentResult( @@ -204,172 +234,6 @@ public sealed class AdminOperationsActor : ReceiveActor msg.CorrelationId)); return; } - - // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject - // on ANY validation error. The config is now canonical — the company overlay loader emits - // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId - // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions, - // …) gates the deploy. A green build in this repo does not prove the config is valid; this - // is the last guard before a bad address space (or a non-derived EquipmentId) ships. - // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can - // (rarely) be a false positive if an external-id reservation release has not yet been - // committed/visible when the snapshot is read — operators seeing a spurious one should - // check ExternalIdReservation state before re-submitting. - var draft = await DraftSnapshotFactory.FromConfigDbAsync(db); - var errors = DraftValidator.Validate(draft).ToList(); - - // Cluster-topology guard. The SQL - // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode) - // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an - // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at - // Hot/Warm and the constraint stays green, which would boot the runtime into an - // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of - // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows. - // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing - // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary. - // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT - // transaction), not a second query that could observe a later edit. - var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList(); - var nodesByCluster = draft.ClusterNodes - .GroupBy(n => n.ClusterId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); - foreach (var cluster in clusters) - { - var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns) - ? (IReadOnlyList)ns - : []; - errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes)); - } - - // Deploy-time TagConfig strictness inspection over equipment tags. The - // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would - // REJECT — a present-but-invalid enum value, an absent or malformed address field, a - // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads - // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum; - // it now rejects the tag, so these warnings describe rejections.) In Warn mode - // (default) these are non-blocking (logged + appended to the accepted result Message); in Error - // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed. - // Running servers are untouched either way — the gate only sees re-deploys. - // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId / - // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type. - var driverTypeById = draft.DriverInstances - .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal); - var driverByDevice = draft.Devices - .GroupBy(d => d.DeviceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal); - var deviceConfigById = draft.Devices - .GroupBy(d => d.DeviceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal); - var tagConfigWarnings = new List<(string TagId, string Text)>(); - foreach (var t in draft.Tags) - { - if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue; - if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue; - foreach (var w in EquipmentTagConfigInspector.Inspect( - driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId))) - tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}")); - } - if (tagConfigWarnings.Count > 0) - { - if (_tagConfigValidationMode == TagConfigValidationMode.Error) - errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId))); - else - foreach (var x in tagConfigWarnings) - _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text); - } - - if (errors.Count > 0) - { - var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}")); - _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}", - errors.Count, summary); - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Rejected, - DeploymentId: null, - RevisionHash: null, - Message: summary, - msg.CorrelationId)); - return; - } - - // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough) - // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc - // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct - // sources are counted (the compile cache keys on source, so duplicates collapse to one unit). - // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard - // reject; this advisory rides in the Accepted Message so the UI can show it. - // advisory is declared outside the inner try so the Accepted reply below can still read it even - // when the guardrail query throws (e.g. transient SQL). A failed estimate must NEVER block a - // valid deploy — the outer catch is reserved for genuine seal/save failures. - string? advisory = null; - try - { - const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) - // Projected from the snapshot already in hand — Script.SourceCode used to be read three times. - var compiled = draft.Scripts - .Select(s => s.SourceCode) - .Where(src => !PassthroughScript.TryMatch(src, out _)) - .Distinct(StringComparer.Ordinal) - .Count(); - if (compiled > 0) - { - var estMiB = compiled * PerScriptMiB; - _log.Warning( - "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it", - compiled, estMiB); - advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)"; - } - } - catch (Exception ex) - { - // Guardrail is advisory-only — a failed estimate must never block a valid deploy. - _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped"); - advisory = null; - } - - // Composed from the validated snapshot, never from a second read: an edit that lands between - // validation and here is simply not in this artifact (R03:3.2). - var artifact = ConfigComposer.Compose(draft); - var deploymentId = DeploymentId.NewId(); - var revHash = RevisionHash.Parse(artifact.RevisionHash); - - db.Deployments.Add(new Deployment - { - DeploymentId = deploymentId.Value, - RevisionHash = artifact.RevisionHash, - Status = DeploymentStatus.Dispatching, - CreatedBy = msg.CreatedBy, - ArtifactBlob = artifact.Blob, - }); - - // Marker ConfigEdit row so the audit timeline shows the deployment snapshot. - db.ConfigEdits.Add(new ConfigEdit - { - EntityType = "Deployment", - EntityId = deploymentId.Value, - FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}", - EditedBy = msg.CreatedBy, - SourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown", - }); - - await db.SaveChangesAsync(); - - _coordinator.Tell(new DispatchDeployment(deploymentId, revHash, msg.CorrelationId)); - - // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so - // the operator sees the non-blocking strictness findings on a successful deploy. - var messageParts = new List(); - if (advisory is not null) messageParts.Add(advisory); - messageParts.AddRange(tagConfigWarnings.Select(x => x.Text)); - var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null; - - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Accepted, - deploymentId, - revHash, - Message: acceptedMessage, - msg.CorrelationId)); } catch (Exception ex) { @@ -380,10 +244,233 @@ public sealed class AdminOperationsActor : ReceiveActor RevisionHash: null, Message: ex.Message, msg.CorrelationId)); + return; } + + _composeInFlight = msg; + // Captured here: Context/Sender are not available on the thread-pool thread. + var sourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown"; + var self = Self; + Task.Run(() => ComposeDeploymentAsync(msg, sourceNode)) + .PipeTo(self, + success: composed => new DeployComposed(msg, replyTo, composed.Result, composed.Dispatch), + failure: ex => new DeployComposeFailed(msg, replyTo, ex)); } - private async Task HandleTestDriverConnectAsync(TestDriverConnect msg) + private void HandleDeployComposed(DeployComposed msg) + { + _composeInFlight = null; + // The row is committed (the compose task completed), so the coordinator can be told now; it reads + // the row back by id when it seeds the expected-ack set. + if (msg.Dispatch is { } dispatch) _coordinator.Tell(dispatch); + msg.ReplyTo.Tell(msg.Result); + } + + private void HandleDeployComposeFailed(DeployComposeFailed msg) + { + _composeInFlight = null; + _log.Error(msg.Error, "StartDeployment failed for {CreatedBy}", msg.Request.CreatedBy); + msg.ReplyTo.Tell(new StartDeploymentResult( + StartDeploymentOutcome.Rejected, + DeploymentId: null, + RevisionHash: null, + Message: msg.Error.Message, + msg.Request.CorrelationId)); + } + + /// Outcome of the off-actor compose: the reply to send, and the dispatch to issue when Accepted. + private sealed record ComposeOutcome(StartDeploymentResult Result, DispatchDeployment? Dispatch); + + private sealed record DeployComposed( + StartDeployment Request, IActorRef ReplyTo, StartDeploymentResult Result, DispatchDeployment? Dispatch); + + private sealed record DeployComposeFailed(StartDeployment Request, IActorRef ReplyTo, Exception Error); + + /// + /// The deploy compose, off the actor thread. Touches no actor state: everything it needs is in + /// , and the readonly fields captured at + /// construction (, , — + /// Akka's logging adapter is thread-safe). + /// + private async Task ComposeDeploymentAsync(StartDeployment msg, string sourceNode) + { + await using var db = await _dbFactory.CreateDbContextAsync(); + + // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject + // on ANY validation error. The config is now canonical — the company overlay loader emits + // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId + // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions, + // …) gates the deploy. A green build in this repo does not prove the config is valid; this + // is the last guard before a bad address space (or a non-derived EquipmentId) ships. + // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can + // (rarely) be a false positive if an external-id reservation release has not yet been + // committed/visible when the snapshot is read — operators seeing a spurious one should + // check ExternalIdReservation state before re-submitting. + var draft = await DraftSnapshotFactory.FromConfigDbAsync(db); + var errors = DraftValidator.Validate(draft).ToList(); + + // Cluster-topology guard. The SQL + // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode) + // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an + // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at + // Hot/Warm and the constraint stays green, which would boot the runtime into an + // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of + // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows. + // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing + // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary. + // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT + // transaction), not a second query that could observe a later edit. + var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList(); + var nodesByCluster = draft.ClusterNodes + .GroupBy(n => n.ClusterId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); + foreach (var cluster in clusters) + { + var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns) + ? (IReadOnlyList)ns + : []; + errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes)); + } + + // Deploy-time TagConfig strictness inspection over equipment tags. The + // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would + // REJECT — a present-but-invalid enum value, an absent or malformed address field, a + // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads + // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum; + // it now rejects the tag, so these warnings describe rejections.) In Warn mode + // (default) these are non-blocking (logged + appended to the accepted result Message); in Error + // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed. + // Running servers are untouched either way — the gate only sees re-deploys. + // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId / + // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type. + var driverTypeById = draft.DriverInstances + .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal); + var driverByDevice = draft.Devices + .GroupBy(d => d.DeviceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal); + var deviceConfigById = draft.Devices + .GroupBy(d => d.DeviceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal); + var tagConfigWarnings = new List<(string TagId, string Text)>(); + foreach (var t in draft.Tags) + { + if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue; + if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue; + foreach (var w in EquipmentTagConfigInspector.Inspect( + driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId))) + tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}")); + } + if (tagConfigWarnings.Count > 0) + { + if (_tagConfigValidationMode == TagConfigValidationMode.Error) + errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId))); + else + foreach (var x in tagConfigWarnings) + _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text); + } + + if (errors.Count > 0) + { + var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}")); + _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}", + errors.Count, summary); + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.Rejected, + DeploymentId: null, + RevisionHash: null, + Message: summary, + msg.CorrelationId), + Dispatch: null); + } + + // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough) + // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc + // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct + // sources are counted (the compile cache keys on source, so duplicates collapse to one unit). + // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard + // reject; this advisory rides in the Accepted Message so the UI can show it. + // advisory is declared outside the inner try so the Accepted reply below can still read it even + // when the guardrail throws. A failed estimate must NEVER block a valid deploy — the caller's + // failure branch is reserved for genuine seal/save failures. + string? advisory = null; + try + { + const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) + // Projected from the snapshot already in hand — Script.SourceCode used to be read three times. + var compiled = draft.Scripts + .Select(s => s.SourceCode) + .Where(src => !PassthroughScript.TryMatch(src, out _)) + .Distinct(StringComparer.Ordinal) + .Count(); + if (compiled > 0) + { + var estMiB = compiled * PerScriptMiB; + _log.Warning( + "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it", + compiled, estMiB); + advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)"; + } + } + catch (Exception ex) + { + // Guardrail is advisory-only — a failed estimate must never block a valid deploy. + _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped"); + advisory = null; + } + + // Composed from the validated snapshot, never from a second read: an edit that lands between + // validation and here is simply not in this artifact (R03:3.2). + var artifact = ConfigComposer.Compose(draft); + var deploymentId = DeploymentId.NewId(); + var revHash = RevisionHash.Parse(artifact.RevisionHash); + + db.Deployments.Add(new Deployment + { + DeploymentId = deploymentId.Value, + RevisionHash = artifact.RevisionHash, + Status = DeploymentStatus.Dispatching, + CreatedBy = msg.CreatedBy, + ArtifactBlob = artifact.Blob, + }); + + // Marker ConfigEdit row so the audit timeline shows the deployment snapshot. + db.ConfigEdits.Add(new ConfigEdit + { + EntityType = "Deployment", + EntityId = deploymentId.Value, + FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}", + EditedBy = msg.CreatedBy, + SourceNode = sourceNode, + }); + + await db.SaveChangesAsync(); + + // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so + // the operator sees the non-blocking strictness findings on a successful deploy. + var messageParts = new List(); + if (advisory is not null) messageParts.Add(advisory); + messageParts.AddRange(tagConfigWarnings.Select(x => x.Text)); + var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null; + + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.Accepted, + deploymentId, + revHash, + Message: acceptedMessage, + msg.CorrelationId), + new DispatchDeployment(deploymentId, revHash, msg.CorrelationId)); + } + + /// + /// Test Connect runs on the thread pool and pipes its reply straight to the requester: a probe + /// against a black-holed PLC (60 s clamp) must not suspend the singleton's mailbox (R03:3.1). Every + /// path inside the task yields a , so the requester never + /// sees a Status.Failure. + /// + private void HandleTestDriverConnect(TestDriverConnect msg) { var replyTo = Sender; if (!_probesByType.TryGetValue(msg.DriverType, out var probe)) @@ -396,6 +483,11 @@ public sealed class AdminOperationsActor : ReceiveActor return; } + Task.Run(() => ProbeAsync(probe, msg)).PipeTo(replyTo); + } + + private async Task ProbeAsync(IDriverProbe probe, TestDriverConnect msg) + { var clampedSec = Math.Clamp(msg.TimeoutSeconds, 1, 60); var timeout = TimeSpan.FromSeconds(clampedSec); using var cts = new CancellationTokenSource(timeout); @@ -405,28 +497,28 @@ public sealed class AdminOperationsActor : ReceiveActor { var result = await probe.ProbeAsync(msg.ConfigJson, timeout, cts.Token); sw.Stop(); - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( result.Ok, result.Message, result.Ok ? sw.Elapsed.TotalMilliseconds : (double?)null, - msg.CorrelationId)); + msg.CorrelationId); } catch (OperationCanceledException) { - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( false, $"Probe timed out after {clampedSec}s.", null, - msg.CorrelationId)); + msg.CorrelationId); } catch (Exception ex) { _log.Error(ex, "Probe for {DriverType} threw", msg.DriverType); - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( false, ex.Message, null, - msg.CorrelationId)); + msg.CorrelationId); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs new file mode 100644 index 00000000..95a2b0fc --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs @@ -0,0 +1,159 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:3.1 — the admin singleton's mailbox used to be suspended for the whole of every +/// ReceiveAsync: one Test Connect against a black-holed PLC (60 s clamp) or one large deploy +/// compose stalled every other admin operation fleet-wide, alarm acks included. Both now run off the +/// actor thread; only the "is one in flight" check stays serialised. +/// +public sealed class AdminOperationsActorMailboxTests : ControlPlaneActorTestBase +{ + private static readonly TimeSpan AckBudget = TimeSpan.FromSeconds(1); + + /// An alarm ack completes in under a second while a 60 s Test Connect is still pending. + [Fact] + public void Alarm_ack_completes_while_a_60s_test_connect_is_pending() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe("coord"); + var probe = new BlockingProbe(); + var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe })); + var connectClient = CreateTestProbe("connect-client"); + + connectClient.Send(actor, new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 60, Guid.NewGuid())); + probe.Started.Wait(PresenceBudget).ShouldBeTrue("the probe must be in flight before the ack is sent"); + + actor.Tell(new AcknowledgeAlarmCommand("alarm-1", "op", null, CorrelationId.NewId())); + var ack = ExpectMsg(AckBudget); + ack.Ok.ShouldBeTrue(); + + // Positive control: the probe reply still lands, with the right sender, once the probe returns. + connectClient.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + probe.Release(new DriverProbeResult(true, null, TimeSpan.FromMilliseconds(5))); + var result = connectClient.ExpectMsg(PresenceBudget); + result.Ok.ShouldBeTrue(); + result.LatencyMs.ShouldNotBeNull(); + } + + /// A probe that times out still answers its own sender with the clamped timeout message. + [Fact] + public void Test_connect_timeout_is_reported_to_its_sender() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe("coord"); + var probe = new BlockingProbe(); // never released: the CTS fires at the 1 s clamp floor + var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe })); + + actor.Tell(new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 1, Guid.NewGuid())); + + var result = ExpectMsg(PresenceBudget); + result.Ok.ShouldBeFalse(); + result.Message.ShouldNotBeNull(); + result.Message.ShouldContain("timed out"); + } + + /// + /// The deploy compose (snapshot + validate + gzip + row write) runs off the actor: an alarm ack + /// completes while the compose is parked on a gated DB context, and the deploy still completes + /// (Accepted + dispatched) once the gate opens. + /// + [Fact] + public void Alarm_ack_completes_while_a_deploy_compose_is_in_flight() + { + var gated = new GatedDbFactory(NewInMemoryDbFactory()); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty())); + var deployer = CreateTestProbe("deployer"); + + gated.Hold(); + deployer.Send(actor, new StartDeployment("joe", CorrelationId.NewId())); + gated.Entered.Wait(PresenceBudget).ShouldBeTrue("the compose must be parked before the ack is sent"); + + actor.Tell(new AcknowledgeAlarmCommand("alarm-2", "op", null, CorrelationId.NewId())); + ExpectMsg(AckBudget).Ok.ShouldBeTrue(); + + // Positive control: the deploy completes once released. + deployer.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + gated.Release(); + deployer.ExpectMsg(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.ExpectMsg(PresenceBudget); + } + + /// + /// Only the in-flight check is serialised — and it must still catch a second request that arrives + /// while the first compose has not yet written its row (the DB alone would let both through). + /// + [Fact] + public void Second_StartDeployment_during_an_in_flight_compose_is_refused() + { + var gated = new GatedDbFactory(NewInMemoryDbFactory()); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty())); + var first = CreateTestProbe("first"); + var second = CreateTestProbe("second"); + + gated.Hold(); + first.Send(actor, new StartDeployment("joe", CorrelationId.NewId())); + gated.Entered.Wait(PresenceBudget).ShouldBeTrue(); + + second.Send(actor, new StartDeployment("kim", CorrelationId.NewId())); + second.ExpectMsg(AckBudget).Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight); + + gated.Release(); + first.ExpectMsg(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.ExpectMsg(PresenceBudget); + } + + private sealed class BlockingProbe : IDriverProbe + { + private readonly TaskCompletionSource _tcs = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ManualResetEventSlim Started { get; } = new(); + public string DriverType => "Blocking"; + + public async Task ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct) + { + Started.Set(); + return await _tcs.Task.WaitAsync(ct); + } + + public void Release(DriverProbeResult result) => _tcs.TrySetResult(result); + } + + /// + /// Passes synchronous context creation through (the serialised in-flight check) and parks the + /// asynchronous one (the off-actor compose) on a gate while is in effect. + /// + private sealed class GatedDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private volatile TaskCompletionSource? _gate; + + public ManualResetEventSlim Entered { get; } = new(); + + public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _gate?.TrySetResult(); + + public OtOpcUaConfigDbContext CreateDbContext() => inner.CreateDbContext(); + + public async Task CreateDbContextAsync(CancellationToken cancellationToken = default) + { + Entered.Set(); + if (_gate is { } gate) await gate.Task.WaitAsync(cancellationToken); + return inner.CreateDbContext(); + } + } +} From f5579e6ef2ba48358814ea93f143d276d9a72867 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:15:05 -0400 Subject: [PATCH 04/19] fix(host): make a failed OPC UA SDK start visible and stop the SDK on StopAsync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed SDK start (port in use, PKI unwritable, bad OpcUa:* override) was one Error line: the node stayed /health/ready 200, joined its mesh, could be elected driver Primary and acked deploys through the Null sink while every client got connection-refused. - IOpcUaEndpointState / OpcUaEndpointState: the start outcome is a fact the hosted service maintains (NotStarted / Started / Failed / Stopped + reason). - OpcUaEndpointHealthCheck ("opcua-endpoint", Ready + Active) is Unhealthy unless Started; registered by AddOtOpcUaHealth(hasAdmin, hasDriver) on driver-role nodes. - A driver-only node rethrows the start failure (Program.cs's fatal handler exits non-zero so the supervisor restarts it); a fused node keeps its AdminUI up but is visibly not ready. - StopAsync detaches the sinks and then STOPS the SDK (the port closes, clients get connection-refused rather than a stale, inert endpoint); DisposeAsync is idempotent with it. - The ServiceLevel calculator / redundancy election live in Cluster + Runtime (not this lane); the fact is exposed for the Wave 3 follow-up to consume. Tests: port-in-use ⇒ state Failed, probe Unhealthy, real /health/ready and /health/active 503 (positive control: 200 once Started); driver-only rethrows; StopAsync closes the port and DisposeAsync twice is a no-op; the probe registration is on both tiers for driver nodes only. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Health/HealthEndpoints.cs | 21 +- .../Health/OpcUaEndpointHealthCheck.cs | 45 ++++ .../OpcUa/IOpcUaEndpointState.cs | 73 ++++++ .../OpcUa/OtOpcUaServerHostedService.cs | 124 ++++++++-- src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 10 +- .../Health/ActiveTierRegistrationTests.cs | 28 +++ ...cUaServerHostedServiceStartFailureTests.cs | 213 ++++++++++++++++++ 7 files changed, 492 insertions(+), 22 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs index f7a740bd..2f57158f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs @@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Host.Configuration; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; namespace ZB.MOM.WW.OtOpcUa.Host.Health; @@ -28,11 +29,29 @@ public static class HealthEndpoints /// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to /// true so admin/fused callers (including the test harnesses) keep the probe unchanged. /// + /// + /// Whether this node carries the driver role and therefore serves an OPC UA endpoint. When + /// true the joins the ready + active tiers (R03:2.1), reading + /// the the host registers alongside OtOpcUaServerHostedService. + /// Defaults to false so admin-only callers and the existing harnesses register nothing new. + /// /// The same service collection, for chaining. - public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true) + public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false) { var checks = services.AddHealthChecks(); + // A driver node's whole purpose is its OPC UA endpoint, so a failed SDK start must take it out of + // the ready AND active tiers: readiness because it cannot serve, active because a load balancer or + // the fleet must not route to it. Factory-registered so the state singleton is resolved per probe. + if (hasDriver) + { + checks.Add(new HealthCheckRegistration( + OpcUaEndpointHealthCheck.Name, + sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService()), + failureStatus: null, + tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })); + } + // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config // from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so // the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs new file mode 100644 index 00000000..8caa69f5 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Health; + +/// +/// Ready + Active tier probe for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is +/// listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy +/// one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets +/// connection-refused. +/// +/// +/// Registered on driver-role nodes only (); an +/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. +/// +public sealed class OpcUaEndpointHealthCheck : IHealthCheck +{ + /// The registration name, as it appears in the health JSON. + public const string Name = "opcua-endpoint"; + + private readonly IOpcUaEndpointState _state; + + /// Initializes a new instance of the class. + /// The endpoint lifecycle fact the hosted service maintains. + public OpcUaEndpointHealthCheck(IOpcUaEndpointState state) + { + ArgumentNullException.ThrowIfNull(state); + _state = state; + } + + /// + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + var data = new Dictionary { ["status"] = _state.Status.ToString() }; + var result = _state.Status switch + { + OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data), + OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data), + OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data), + _ => HealthCheckResult.Unhealthy( + "OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data), + }; + return Task.FromResult(result); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs new file mode 100644 index 00000000..c2377ea1 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs @@ -0,0 +1,73 @@ +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// Lifecycle of this node's OPC UA endpoint as drives it. +public enum OpcUaEndpointStatus +{ + /// The hosted service has not run StartAsync yet. + NotStarted, + + /// The SDK is listening and the address-space / ServiceLevel sinks are bound. + Started, + + /// The SDK failed to start (port in use, PKI unwritable, bad OpcUa:* override, …). + Failed, + + /// The hosted service stopped the SDK; the port is released. + Stopped, +} + +/// +/// The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed +/// start was one Error line: /health/ready stayed 200, the node joined its mesh, could be +/// elected driver Primary and acked deploys as Applied through the Null sink — while clients got +/// connection-refused from the node the fleet called healthy. +/// +/// +/// Consumed by OpcUaEndpointHealthCheck (Ready + Active tiers). The redundancy election and the +/// ServiceLevel calculator do not read it yet — they live outside the Host (Cluster / +/// Runtime); feeding them is the Wave 3 follow-up the lane report names. Until then a fused +/// node whose SDK failed is visible (503) but still Primary-eligible. +/// +public interface IOpcUaEndpointState +{ + /// The endpoint's current lifecycle status. + OpcUaEndpointStatus Status { get; } + + /// Why the start failed, when is . + string? FailureReason { get; } +} + +/// Mutable ; written by the hosted service, read by the health check. +public sealed class OpcUaEndpointState : IOpcUaEndpointState +{ + private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted; + private volatile string? _failureReason; + + /// + public OpcUaEndpointStatus Status => _status; + + /// + public string? FailureReason => _failureReason; + + /// Records a successful SDK start. + public void MarkStarted() + { + _failureReason = null; + _status = OpcUaEndpointStatus.Started; + } + + /// Records a failed SDK start. + /// The failure, as the health probe should report it. + public void MarkFailed(string reason) + { + _failureReason = reason; + _status = OpcUaEndpointStatus.Failed; + } + + /// Records that the hosted service stopped the SDK. + public void MarkStopped() + { + _failureReason = null; + _status = OpcUaEndpointStatus.Stopped; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs index 79f62cd8..609b0087 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Authorization; @@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl private readonly IDriverPrimaryRoleSource? _primaryRoleSource; private readonly HistorianUpdaterHolder _historianUpdater; private readonly IHistoryUpdateRoleView? _historyUpdateRoleView; + private readonly OpcUaEndpointState _endpointState; + private readonly bool _rethrowStartFailure; private OpcUaApplicationHost? _appHost; private OtOpcUaSdkServer? _server; @@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl /// The HistoryUpdate verdict DriverHostActor publishes (the /// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every /// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow). + /// The endpoint lifecycle fact this service maintains for the + /// opcua-endpoint health probe (R03:2.1). Optional: a private instance is kept when none is registered, + /// so a harness that never probes health is unchanged. + /// The node's resolved roles. A node with the driver role and NO admin role exists only + /// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and + /// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through + /// the health probe instead. Null (harnesses) behaves as fused. public OtOpcUaServerHostedService( IOptions options, DeferredAddressSpaceSink deferredSink, @@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl ILoggerFactory loggerFactory, IDriverPrimaryRoleSource? primaryRoleSource = null, HistorianUpdaterHolder? historianUpdater = null, - IHistoryUpdateRoleView? historyUpdateRoleView = null) + IHistoryUpdateRoleView? historyUpdateRoleView = null, + OpcUaEndpointState? endpointState = null, + EffectiveRoles? roles = null) { ArgumentNullException.ThrowIfNull(aclOptions); ArgumentNullException.ThrowIfNull(time); @@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl _primaryRoleSource = primaryRoleSource; _historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None; _historyUpdateRoleView = historyUpdateRoleView; + _endpointState = endpointState ?? new OpcUaEndpointState(); + _rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false }; _logger = loggerFactory.CreateLogger(); } @@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl } catch (Exception ex) { + // R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy + // (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits + // so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct + // the OpcUa:* config without a process bounce — but it is visibly not ready. + _endpointState.MarkFailed(ex.Message); + await ReleaseFailedStartAsync().ConfigureAwait(false); + if (_rethrowStartFailure) + { + _logger.LogCritical(ex, + "OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " + + "node justifies staying up; failing host start so the supervisor restarts the process"); + throw; + } _logger.LogError(ex, - "OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op"); - // Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot. - // Operators see the failure via the logs + can correct config without a process bounce - // of the whole binary. + "OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " + + "OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section"); return; } @@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl { _logger.LogWarning( "OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null"); + _endpointState.MarkFailed("SDK reported started but the node manager is null"); return; } @@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl { _logger.LogError(ex, "OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues"); + _endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message); try { - await StopAsync(cancellationToken).ConfigureAwait(false); + DetachSinks(); } catch (Exception cleanupError) { _logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure"); } - var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync( - async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false), - () => _appHost.DisposeAsync(), _logger).ConfigureAwait(false); - if (released) - { - _appHost = null; - _server = null; - } + await ReleaseEndpointAsync().ConfigureAwait(false); return; } @@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // attached, so the first tick cannot land on a half-wired server. StartRevalidationTimer(); + _endpointState.MarkStarted(); _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound"); } + /// + /// Releases an SDK whose StartAsync threw. The application instance may hold a half-opened + /// listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the + /// transport is up); disposing it is the only way to be sure. The state stays Failed. + /// + private async Task ReleaseFailedStartAsync() + { + var appHost = _appHost; + _appHost = null; + _server = null; + if (appHost is null) return; + try + { + await appHost.DisposeAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw"); + } + } + + /// + /// Stops the SDK and disposes the application host, releasing endpoint ownership only when both + /// succeeded (): on a failure the + /// fields are retained so retries at host disposal. + /// + private async Task ReleaseEndpointAsync() + { + var server = _server; + var appHost = _appHost; + if (server is null && appHost is null) return; + var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync( + async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); }, + () => appHost?.DisposeAsync() ?? ValueTask.CompletedTask, + _logger).ConfigureAwait(false); + if (released) + { + _appHost = null; + _server = null; + } + } + /// /// The ServerHistorian:HonorReturnBounds kill switch StartAsync applies to the node manager /// (default true when the section or key is absent). Exposed so the wiring is testable without booting the SDK. @@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl } /// - /// Stops the OPC UA server asynchronously. + /// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes. /// + /// + /// R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown, + /// the other hosted services' StopAsync), serving the last-written values with an emptied authority: + /// reads succeeded on stale data while writes got "unavailable", and clients saw a server that was + /// up but inert instead of the connection-refused that triggers their failover. + /// stays idempotent with this: ownership is nulled once the stop succeeded. + /// /// Cancellation token. /// A task that represents the asynchronous operation. - public Task StopAsync(CancellationToken cancellationToken) + public async Task StopAsync(CancellationToken cancellationToken) + { + DetachSinks(); + await ReleaseEndpointAsync().ConfigureAwait(false); + if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped(); + } + + /// + /// Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor + /// or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running + /// until . + /// + private void DetachSinks() { // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose // authority and gate have already been cleared is the shape this whole teardown block exists to @@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // request rather than let the seams fall back to their role checks — a stopping node should shed // enforcement in the same order it acquired it. _server?.SetDataPlaneAuthorizer(null); - return Task.CompletedTask; } /// @@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl public async ValueTask DisposeAsync() { // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must - // not leave a tick armed against a server that is about to go away. + // not leave a tick armed against a server that is about to go away. A StopAsync that released the + // endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left + // them set, and this is the retry. _revalidationTimer?.Dispose(); _revalidationTimer = null; - if (_appHost is not null) await _appHost.DisposeAsync(); + var appHost = _appHost; + _appHost = null; + _server = null; + if (appHost is not null) await appHost.DisposeAsync(); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 6404504e..d326d1b1 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -452,6 +452,11 @@ if (hasDriver) // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once. builder.Services.TryAddSingleton>(sp => () => sp.GetRequiredService()); + // The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint + // health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the + // interface, so the writer and the probe cannot disagree. + builder.Services.AddSingleton(); + builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); } @@ -554,8 +559,9 @@ if (hasAdmin) builder.Services.AddOtOpcUaSecrets(builder.Configuration); // hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so -// probing it would throw when the readiness endpoint resolves the factory. -builder.Services.AddOtOpcUaHealth(hasAdmin); +// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the +// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green. +builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver); builder.Services.AddOtOpcUaObservability(builder.Configuration); // gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs index 1a9730ce..d880e7b3 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs @@ -49,6 +49,34 @@ public sealed class ActiveTierRegistrationTests ClusterPrimaryRegistration(hasAdmin).Tags.ShouldContain(ZbHealthTags.Active); } + /// + /// R03:2.1 — the OPC UA endpoint probe is on BOTH tiers of a driver node (it cannot serve, and must not + /// be routed to) and absent from an admin-only node, which starts no endpoint to be degraded by. + /// + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Opcua_endpoint_probe_is_registered_on_ready_and_active_for_driver_nodes_only(bool hasDriver) + { + var registrations = new ServiceCollection() + .AddOtOpcUaHealth(hasAdmin: true, hasDriver: hasDriver) + .BuildServiceProvider() + .GetRequiredService>() + .Value.Registrations + .Where(r => r.Name == OpcUaEndpointHealthCheck.Name) + .ToList(); + + if (!hasDriver) + { + registrations.ShouldBeEmpty(); + return; + } + + var registration = registrations.ShouldHaveSingleItem(); + registration.Tags.ShouldContain(ZbHealthTags.Ready); + registration.Tags.ShouldContain(ZbHealthTags.Active); + } + [Fact] public void Active_tier_check_is_not_on_the_ready_tier() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs new file mode 100644 index 00000000..341595d7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs @@ -0,0 +1,213 @@ +using System.Net; +using System.Net.Sockets; +using Akka.Hosting; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.Health; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// R03:2.1 / 1.4 — a failed OPC UA SDK start used to be one Error line: the node stayed Ready, joined +/// its mesh, and acked deploys through the Null sink while clients got connection-refused. The start +/// outcome is now a fact () the ready/active tiers read, a driver-only +/// node rethrows so the process exits, and StopAsync actually stops the SDK. +/// +public sealed class OtOpcUaServerHostedServiceStartFailureTests +{ + private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30); + + /// + /// Port already bound ⇒ the SDK cannot listen ⇒ the state is Failed, the probe is Unhealthy and the + /// real /health/ready endpoint answers 503 — while StartAsync itself returns (fused node). + /// Positive control: the same endpoint answers 200 once the state is Started. + /// + [Fact] + public async Task Port_in_use_marks_the_endpoint_failed_and_health_ready_answers_503() + { + using var deadline = new CancellationTokenSource(Deadline); + using var squatter = SquatAllInterfaces(out var port); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var fused = EffectiveRoles.Resolve(new[] { "admin", "driver" }, envRoles: null); + + var service = NewService(port, pki, state, fused); + try + { + await service.StartAsync(deadline.Token); // must NOT throw on a fused node + + state.Status.ShouldBe(OpcUaEndpointStatus.Failed); + state.FailureReason.ShouldNotBeNullOrWhiteSpace(); + + var probe = await new OpcUaEndpointHealthCheck(state).CheckHealthAsync(new HealthCheckContext()); + probe.Status.ShouldBe(HealthStatus.Unhealthy); + probe.Description.ShouldNotBeNull(); + probe.Description.ShouldContain("failed to start"); + + await using var web = BuildHealthHost(state); + await web.StartAsync(deadline.Token); + using var http = new HttpClient { BaseAddress = new Uri(web.Urls.First()) }; + + (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + (await http.GetAsync("/health/active", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + + // Positive control on the endpoint: the same host answers 200 for a started endpoint, so the + // 503 above measured the probe and not a broken health pipeline. + state.MarkStarted(); + (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.OK); + await web.StopAsync(deadline.Token); + } + finally + { + await service.DisposeAsync(); + squatter.Stop(); + Cleanup(pki); + } + } + + /// A driver-only node has nothing else to stay up for: the failed start propagates. + [Fact] + public async Task Driver_only_node_rethrows_when_the_sdk_fails_to_start() + { + using var deadline = new CancellationTokenSource(Deadline); + using var squatter = SquatAllInterfaces(out var port); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var driverOnly = EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null); + + var service = NewService(port, pki, state, driverOnly); + try + { + await Should.ThrowAsync(() => service.StartAsync(deadline.Token)); + state.Status.ShouldBe(OpcUaEndpointStatus.Failed); + } + finally + { + await service.DisposeAsync(); + squatter.Stop(); + Cleanup(pki); + } + } + + /// + /// A clean start is Started and listening; StopAsync closes the port (connection refused, not + /// a stale-but-listening endpoint) and marks Stopped; DisposeAsync twice afterwards is a no-op. + /// + [Fact] + public async Task StopAsync_closes_the_port_and_DisposeAsync_is_idempotent() + { + using var deadline = new CancellationTokenSource(Deadline); + var port = FreePort(); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var service = NewService(port, pki, state, EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null)); + try + { + await service.StartAsync(deadline.Token); + state.Status.ShouldBe(OpcUaEndpointStatus.Started); + using (var client = new TcpClient()) + { + await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token); + client.Connected.ShouldBeTrue("positive control: the endpoint is listening after StartAsync"); + } + + await service.StopAsync(deadline.Token); + + state.Status.ShouldBe(OpcUaEndpointStatus.Stopped); + using (var client = new TcpClient()) + { + await Should.ThrowAsync(async () => await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token)); + } + + await service.DisposeAsync(); + await service.DisposeAsync(); // idempotent + } + finally + { + await service.DisposeAsync(); + Cleanup(pki); + } + } + + private static OtOpcUaServerHostedService NewService(int port, string pki, OpcUaEndpointState state, EffectiveRoles roles) => + new( + Options.Create(new OpcUaApplicationHostOptions + { + ApplicationName = "OtOpcUa.StartFailure", + ApplicationUri = $"urn:OtOpcUa.StartFailure:{Guid.NewGuid():N}", + OpcUaPort = port, PublicHostname = "localhost", PkiStoreRoot = pki, + }), + new DeferredAddressSpaceSink(), new DeferredServiceLevelPublisher(), + NullOpcUaUserAuthenticator.Instance, NullHistorianDataSource.Instance, + EmptyDataPlaneAuthoritySource.Instance, new ConfigurationBuilder().Build(), + () => throw new NotSupportedException("no actor system required"), new ActorRegistry(), + null, Options.Create(new AclOptions()), TimeProvider.System, NullLoggerFactory.Instance, + endpointState: state, roles: roles); + + /// The production health registration (driver role) over the supplied state, on a loopback Kestrel. + private static WebApplication BuildHealthHost(OpcUaEndpointState state) + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + builder.Logging.ClearProviders(); + builder.Services.AddSingleton(state); + builder.Services.AddOtOpcUaHealth(hasAdmin: false, hasDriver: true); + // The akka + cluster-primary probes need a running ActorSystem; keep only the probe under test so + // the status measured is the endpoint's. (The registration shape itself is pinned separately.) + builder.Services.Configure(o => + { + foreach (var r in o.Registrations.Where(r => r.Name != OpcUaEndpointHealthCheck.Name).ToList()) + o.Registrations.Remove(r); + }); + var app = builder.Build(); + app.MapOtOpcUaHealth(); + return app; + } + + /// + /// Holds a port on the IPv4 AND IPv6 wildcard addresses. The SDK binds its listener on the + /// wildcards, and on macOS a loopback-only squatter does not conflict with a wildcard bind (both + /// sockets carry SO_REUSEADDR), so a loopback squatter would let the SDK start — the first run of + /// this test proved exactly that. The SDK also only fails when NEITHER family binds. + /// + private static TcpListener SquatAllInterfaces(out int port) + { + var squatter = new TcpListener(IPAddress.IPv6Any, 0); + squatter.Server.DualMode = true; + squatter.ExclusiveAddressUse = true; + squatter.Start(); + port = ((IPEndPoint)squatter.LocalEndpoint).Port; + return squatter; + } + + private static int FreePort() + { + using var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + var port = ((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); + return port; + } + + private static string NewPki() => Path.Combine(Path.GetTempPath(), $"otopcua-startfail-{Guid.NewGuid():N}"); + + private static void Cleanup(string pki) + { + if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true); + } +} From f72af76e99e241ad1d36b90a8d0952c7d74a17d1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:23:14 -0400 Subject: [PATCH 05/19] fix(security): validate the API-key pepper at start, finish JWT bearer, circuit-aware audit actor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ApiKeyOptionsValidator (ValidateOnStart on any driver/admin node): a blank or placeholder Security:ApiKeys:Pepper fails host start instead of rejecting every key after a green boot (R03:2.3). - JWT is finished and opt-in by key (R03:2.4): with Security:Jwt:SigningKey set, a Bearer scheme (JwtBearerAuthenticationHandler over JwtTokenService.TryValidate / BuildValidationParameters, fail closed) accepts the tokens /auth/token mints; the default scheme is a cookie-or-bearer policy scheme and the fallback policy names it, so /api/* and every RequireAuthorization endpoint take a browser session OR a machine token. JwtOptions are validated at start (JwtOptionsValidator). Without a key, /auth/token is not mapped (404, not the former constructor 500) and no bearer scheme exists. A direct AddJwtBearer needs Microsoft.AspNetCore.Authentication.JwtBearer in Directory.Packages.props (not this lane's file) — the handler is a drop-in to swap. - AuthenticationStateAuditActorAccessor is the registered IAuditActorAccessor: the Blazor circuit's principal first, HttpContext as the fallback (R03:2.5). HttpAuditActorAccessor shares its resolver. - CookieAuthenticationStateProvider (unregistered, fire-and-forget ping loop) is deleted (R03:1.5/2.5). - /auth/login logs the LDAP exception at Warning before answering 503 (R03:2.7). - Admin operations reply with a fixed message + correlation id instead of ex.Message (R03:4.1). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdminOperations/AdminOperationsActor.cs | 40 ++++-- .../Configuration/ApiKeyOptionsValidator.cs | 41 ++++++ src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 13 +- .../AuthenticationStateAuditActorAccessor.cs | 75 +++++++++++ .../Audit/HttpAuditActorAccessor.cs | 26 +--- .../Auth/JwtBearerAuthenticationHandler.cs | 99 ++++++++++++++ .../CookieAuthenticationStateProvider.cs | 79 ----------- .../Endpoints/AuthEndpoints.cs | 17 ++- .../Jwt/JwtOptions.cs | 22 +++ .../Jwt/JwtOptionsValidator.cs | 19 +++ .../ServiceCollectionExtensions.cs | 48 +++++-- ...nOperationsActorServerFaultMessageTests.cs | 68 ++++++++++ .../ApiKeyOptionsValidatorTests.cs | 54 ++++++++ ...henticationStateAuditActorAccessorTests.cs | 80 +++++++++++ .../AuthEndpointsIntegrationTests.cs | 67 +++++++++ .../CapturingLoggerProvider.cs | 25 ++++ .../JwtConfigurationGateTests.cs | 127 ++++++++++++++++++ 17 files changed, 773 insertions(+), 127 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs delete mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index 8b1c9ddb..e78c926e 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -89,6 +89,18 @@ public sealed class AdminOperationsActor : ReceiveActor Receive(HandleShelveAlarm); } + /// + /// The reply text for an unexpected server-side failure (R03:4.1): a fixed sentence plus the request's + /// correlation id, never ex.Message — a SqlException or UriFormatException message carries + /// connection-string fragments and host names, and the reply reaches any ConfigEditor browser session. + /// The detail is in the central log under the same correlation id. + /// + /// The admin operation that failed. + /// The request's correlation token, as logged alongside the exception. + /// The operator-facing message. + internal static string ServerFault(string operation, object correlationId) => + $"{operation} failed on the server; see the central node log for correlation {correlationId}."; + /// /// Hands a central→node command to the mesh transport router, or publishes it directly when /// no router is wired. @@ -136,8 +148,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId}", msg.AlarmId); - replyTo.Tell(new AcknowledgeAlarmResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId} (correlation {CorrelationId})", msg.AlarmId, msg.CorrelationId); + replyTo.Tell(new AcknowledgeAlarmResult(false, ServerFault("Acknowledge", msg.CorrelationId), msg.CorrelationId)); } } @@ -183,8 +195,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId}", msg.Kind, msg.AlarmId); - replyTo.Tell(new ShelveAlarmResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId} (correlation {CorrelationId})", msg.Kind, msg.AlarmId, msg.CorrelationId); + replyTo.Tell(new ShelveAlarmResult(false, ServerFault("Shelve", msg.CorrelationId), msg.CorrelationId)); } } @@ -237,12 +249,12 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "StartDeployment failed for {CreatedBy}", msg.CreatedBy); + _log.Error(ex, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.CreatedBy, msg.CorrelationId); replyTo.Tell(new StartDeploymentResult( StartDeploymentOutcome.Rejected, DeploymentId: null, RevisionHash: null, - Message: ex.Message, + Message: ServerFault("StartDeployment", msg.CorrelationId), msg.CorrelationId)); return; } @@ -269,12 +281,12 @@ public sealed class AdminOperationsActor : ReceiveActor private void HandleDeployComposeFailed(DeployComposeFailed msg) { _composeInFlight = null; - _log.Error(msg.Error, "StartDeployment failed for {CreatedBy}", msg.Request.CreatedBy); + _log.Error(msg.Error, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.Request.CreatedBy, msg.Request.CorrelationId); msg.ReplyTo.Tell(new StartDeploymentResult( StartDeploymentOutcome.Rejected, DeploymentId: null, RevisionHash: null, - Message: msg.Error.Message, + Message: ServerFault("StartDeployment", msg.Request.CorrelationId), msg.Request.CorrelationId)); } @@ -513,10 +525,10 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "Probe for {DriverType} threw", msg.DriverType); + _log.Error(ex, "Probe for {DriverType} threw (correlation {CorrelationId})", msg.DriverType, msg.CorrelationId); return new TestDriverConnectResult( false, - ex.Message, + ServerFault("Test Connect", msg.CorrelationId), null, msg.CorrelationId); } @@ -548,8 +560,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId}", msg.DriverInstanceId); - replyTo.Tell(new RestartDriverResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId); + replyTo.Tell(new RestartDriverResult(false, ServerFault("RestartDriver", msg.CorrelationId), msg.CorrelationId)); } } @@ -578,8 +590,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId}", msg.DriverInstanceId); - replyTo.Tell(new ReconnectDriverResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId); + replyTo.Tell(new ReconnectDriverResult(false, ServerFault("ReconnectDriver", msg.CorrelationId), msg.CorrelationId)); } } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs new file mode 100644 index 00000000..eced66a8 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs @@ -0,0 +1,41 @@ +using ZB.MOM.WW.Configuration; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; + +namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; + +/// +/// Fail-fast startup validator for (R03:2.3). The pepper used to be +/// checked only when the first key was verified — ApiKeySecret.Verify throws on an empty one — +/// so a site node deployed without Security__ApiKeys__Pepper booted green, passed health, +/// applied deployments and denied every machine client with the same opaque "Invalid API key" until +/// someone read the log; a central node with a wrong pepper minted keys that could verify nowhere. +/// Every other security section already fails host start on a bad value; this one now does too. +/// +/// +/// Registered for hasDriver || hasAdmin: a driver node needs the pepper to VERIFY keys, an admin +/// node to MINT them. The minimum length is a floor against a placeholder ("x") passing as +/// configured, not a strength claim — the pepper is a server-side secret, not a password. +/// +public sealed class ApiKeyOptionsValidator : OptionsValidatorBase +{ + /// The shortest pepper accepted (characters, after trimming). + public const int MinPepperLength = 16; + + /// + protected override void Validate(ValidationBuilder builder, ApiKeyOptions options) + { + var pepper = options.Pepper?.Trim() ?? string.Empty; + builder.RequireThat(pepper.Length > 0, + $"{ApiKeyOptions.SectionName}:Pepper is required on any node with the driver or admin role — " + + "supply it via Security__ApiKeys__Pepper or a ${secret:} reference. An empty pepper would reject " + + "every API key (driver) or mint keys that can never verify (admin)."); + builder.RequireThat(pepper.Length == 0 || pepper.Length >= MinPepperLength, + $"{ApiKeyOptions.SectionName}:Pepper must be at least {MinPepperLength} characters."); + builder.RequireThat(!string.IsNullOrWhiteSpace(options.Username), + $"{ApiKeyOptions.SectionName}:Username must not be blank."); + builder.RequireThat(options.StoreTimeoutMs >= 0, + $"{ApiKeyOptions.SectionName}:StoreTimeoutMs must be >= 0 (0 disables the deadline)."); + builder.RequireThat(options.StoreCacheSeconds >= 0, + $"{ApiKeyOptions.SectionName}:StoreCacheSeconds must be >= 0 (0 disables the cache)."); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index d326d1b1..b2300ab6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -162,10 +162,15 @@ builder.AddZbSerilog(o => o.ServiceName = "otopcua"); // central over the fetch-and-cache path (ConfigSource:Mode=FetchAndCache). Registering it // unconditionally would throw on a driver-only node, since AddOtOpcUaConfigDb requires the string. // A fused admin+driver node keeps ConfigDb via its admin role — byte-for-byte unaffected. -// bound unconditionally: a driver node needs the pepper to VERIFY keys, an admin node needs it -// to MINT them, and getting this wrong on either side is a silent authentication failure rather than a -// startup error. -builder.Services.Configure(builder.Configuration.GetSection(ApiKeyOptions.SectionName)); +// A driver node needs the pepper to VERIFY keys, an admin node needs it to MINT them, and getting this +// wrong on either side used to be a silent authentication failure rather than a startup error (R03:2.3): +// ApiKeyOptionsValidator now fails host start on a blank or placeholder pepper wherever either role is +// present. A node with neither role (dev-only) has no key path and keeps the plain bind. +if (hasDriver || hasAdmin) + builder.Services.AddValidatedOptions( + builder.Configuration, ApiKeyOptions.SectionName); +else + builder.Services.Configure(builder.Configuration.GetSection(ApiKeyOptions.SectionName)); // Security:Acl:Enabled. Default false; see AclOptions for why the flip must be a config // change rather than a redeploy, and what it swaps rather than adds. builder.Services.Configure(builder.Configuration.GetSection(AclOptions.SectionName)); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs new file mode 100644 index 00000000..d689b85e --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs @@ -0,0 +1,75 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Components.Authorization; +using Microsoft.AspNetCore.Http; +using ZB.MOM.WW.Auth.AspNetCore; + +namespace ZB.MOM.WW.OtOpcUa.Security.Audit; + +/// +/// that reads the Blazor circuit's principal first (R03:2.5). +/// +/// +/// +/// reads IHttpContextAccessor, which is right for a +/// minimal-API request but wrong inside a Blazor Server circuit: after the first render there is no +/// current HttpContext on the circuit's threads, so every audit row written from an interactive +/// page would have been stamped "system". The circuit's identity lives in the scoped +/// , so that is consulted first and the HTTP context is the +/// fallback for non-circuit scopes (where the provider is not registered at all). +/// +/// +/// CurrentActor is synchronous by contract; the server-side provider's +/// is already completed in a +/// circuit, and an incomplete task is treated as "no circuit identity" rather than blocked on. +/// +/// +public sealed class AuthenticationStateAuditActorAccessor : IAuditActorAccessor +{ + private readonly IHttpContextAccessor _httpContextAccessor; + private readonly AuthenticationStateProvider? _authenticationState; + + /// Initializes a new instance of the class. + /// The HTTP context accessor (minimal-API fallback). + /// The circuit's authentication state, when resolved inside a Blazor circuit. + public AuthenticationStateAuditActorAccessor( + IHttpContextAccessor httpContextAccessor, + AuthenticationStateProvider? authenticationState = null) + { + ArgumentNullException.ThrowIfNull(httpContextAccessor); + _httpContextAccessor = httpContextAccessor; + _authenticationState = authenticationState; + } + + /// + public string? CurrentActor + { + get + { + if (_authenticationState is not null) + { + var task = _authenticationState.GetAuthenticationStateAsync(); + if (task.IsCompletedSuccessfully) + { + var actor = ResolveActor(task.Result.User); + if (actor is not null) return actor; + } + } + return ResolveActor(_httpContextAccessor.HttpContext?.User); + } + } + + /// The claim-resolution rule shared with . + /// The principal, or null. + /// The actor string, or null when unauthenticated. + internal static string? ResolveActor(ClaimsPrincipal? user) + { + if (user?.Identity?.IsAuthenticated != true) + return null; + + // Prefer the canonical login-name claim; fall back to the Name claim or + // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name). + return user.FindFirst(ZbClaimTypes.Username)?.Value + ?? user.FindFirst(ZbClaimTypes.Name)?.Value + ?? user.Identity.Name; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs index 9a86339f..529504b3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs @@ -1,5 +1,4 @@ using Microsoft.AspNetCore.Http; -using ZB.MOM.WW.Auth.AspNetCore; namespace ZB.MOM.WW.OtOpcUa.Security.Audit; @@ -17,10 +16,10 @@ namespace ZB.MOM.WW.OtOpcUa.Security.Audit; /// , in that order. /// /// -/// Registered as scoped in -/// so that it correctly follows the request scope used by Blazor Server interactive components -/// and minimal-API endpoints. IHttpContextAccessor is registered by -/// AddOtOpcUaAuth via services.AddHttpContextAccessor(). +/// No longer the registered implementation: is, because +/// a Blazor circuit has no HttpContext after its first render and this accessor would stamp every row +/// written from an interactive page "system". Kept as the explicit HTTP-only variant for non-circuit +/// hosts and tests. /// /// public sealed class HttpAuditActorAccessor : IAuditActorAccessor @@ -35,19 +34,6 @@ public sealed class HttpAuditActorAccessor : IAuditActorAccessor } /// - public string? CurrentActor - { - get - { - var user = _httpContextAccessor.HttpContext?.User; - if (user?.Identity?.IsAuthenticated != true) - return null; - - // Prefer the canonical login-name claim; fall back to the Name claim or - // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name). - return user.FindFirst(ZbClaimTypes.Username)?.Value - ?? user.FindFirst(ZbClaimTypes.Name)?.Value - ?? user.Identity.Name; - } - } + public string? CurrentActor => + AuthenticationStateAuditActorAccessor.ResolveActor(_httpContextAccessor.HttpContext?.User); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs new file mode 100644 index 00000000..cc996758 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs @@ -0,0 +1,99 @@ +using System.Text.Encodings.Web; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Primitives; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; + +namespace ZB.MOM.WW.OtOpcUa.Security.Auth; + +/// The authentication scheme names OtOpcUa registers. +public static class OtOpcUaAuthSchemes +{ + /// + /// The bearer scheme that validates the HS256 tokens /auth/token mints, through + /// . Registered only when + /// . + /// + public const string Bearer = "Bearer"; + + /// + /// The default scheme: a policy scheme that forwards to when the request + /// carries an Authorization: Bearer header and to the cookie scheme otherwise, so one + /// RequireAuthorization() / policy accepts a browser session AND a machine token. + /// + public const string CookieOrBearer = "OtOpcUa.CookieOrBearer"; + + /// Whether the request presents a bearer token (the selector's whole rule). + /// The request. + /// when an Authorization: Bearer … header is present. + public static bool HasBearerToken(HttpContext context) => + TryGetBearerToken(context.Request.Headers.Authorization, out _); + + internal static bool TryGetBearerToken(StringValues authorization, out string token) + { + token = string.Empty; + var header = authorization.ToString(); + const string prefix = "Bearer "; + if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; + token = header[prefix.Length..].Trim(); + return token.Length > 0; + } +} + +/// +/// Validates an Authorization: Bearer token minted by /auth/token (R03:2.4). The JWT +/// feature used to be a dead end: tokens were issued and nothing in the product accepted them. This +/// handler is the consuming side, built on so the validation +/// parameters can never drift from the minting side ( +/// sets RoleClaimType/NameClaimType to the minted keys, so [Authorize(Roles)] and +/// IsInRole resolve). +/// +/// +/// Fail closed: no header ⇒ NoResult (the cookie scheme is not consulted by this handler — the +/// policy scheme selects one or the other); a malformed, expired, tampered or wrong-issuer token ⇒ +/// Fail, never a fall-through to the cookie. A direct AddJwtBearer needs the +/// Microsoft.AspNetCore.Authentication.JwtBearer package, which the central package list does +/// not carry; this handler is ~40 lines over the already-referenced System.IdentityModel.Tokens.Jwt +/// and is a drop-in to replace once that package is pinned. +/// +public sealed class JwtBearerAuthenticationHandler : AuthenticationHandler +{ + private readonly JwtTokenService _jwt; + + /// Initializes a new instance of the class. + /// The scheme options monitor. + /// The logger factory. + /// The URL encoder. + /// The token service whose validation parameters mirror its minting side. + public JwtBearerAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder, + JwtTokenService jwt) + : base(options, logger, encoder) + { + _jwt = jwt; + } + + /// + protected override Task HandleAuthenticateAsync() + { + if (!OtOpcUaAuthSchemes.TryGetBearerToken(Request.Headers.Authorization, out var token)) + return Task.FromResult(AuthenticateResult.NoResult()); + + if (!_jwt.TryValidate(token, out var principal) || principal is null) + return Task.FromResult(AuthenticateResult.Fail("Invalid bearer token.")); + + return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(principal, Scheme.Name))); + } + + /// + protected override Task HandleChallengeAsync(AuthenticationProperties properties) + { + Response.StatusCode = StatusCodes.Status401Unauthorized; + Response.Headers.WWWAuthenticate = OtOpcUaAuthSchemes.Bearer; + return Task.CompletedTask; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs deleted file mode 100644 index 0e42b789..00000000 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System.Net.Http.Json; -using System.Security.Claims; -using Microsoft.AspNetCore.Components.Authorization; -using Microsoft.Extensions.Logging; - -namespace ZB.MOM.WW.OtOpcUa.Security.Blazor; - -/// -/// Blazor Server that snapshots the cookie-backed -/// principal supplied at circuit boot and polls /auth/ping every 60 seconds to detect -/// expiry. Mirrors ScadaLink's CentralUI implementation. -/// -public sealed class CookieAuthenticationStateProvider : AuthenticationStateProvider, IAsyncDisposable -{ - private static readonly TimeSpan PingInterval = TimeSpan.FromSeconds(60); - - private readonly HttpClient _http; - private readonly ILogger _logger; - private readonly CancellationTokenSource _cts = new(); - private ClaimsPrincipal _current; - private Task? _pingLoop; - - /// Initializes a new instance of the CookieAuthenticationStateProvider class. - /// The initial claims principal from circuit boot. - /// The HTTP client for authentication ping requests. - /// The logger for diagnostic messages. - public CookieAuthenticationStateProvider( - ClaimsPrincipal initial, - HttpClient http, - ILogger logger) - { - _current = initial; - _http = http; - _logger = logger; - } - - /// - public override Task GetAuthenticationStateAsync() - { - _pingLoop ??= Task.Run(() => PingLoopAsync(_cts.Token)); - return Task.FromResult(new AuthenticationState(_current)); - } - - private async Task PingLoopAsync(CancellationToken ct) - { - try - { - while (!ct.IsCancellationRequested) - { - await Task.Delay(PingInterval, ct).ConfigureAwait(false); - var resp = await _http.GetAsync("/auth/ping", ct).ConfigureAwait(false); - if (!resp.IsSuccessStatusCode && _current.Identity?.IsAuthenticated == true) - { - _logger.LogInformation("/auth/ping returned {Code}; notifying circuit", (int)resp.StatusCode); - _current = new ClaimsPrincipal(new ClaimsIdentity()); - NotifyAuthenticationStateChanged( - Task.FromResult(new AuthenticationState(_current))); - } - } - } - catch (OperationCanceledException) { /* expected on shutdown */ } - catch (Exception ex) - { - _logger.LogWarning(ex, "Auth ping loop terminated unexpectedly"); - } - } - - /// Cancels the background ping loop, awaits its completion, and disposes the cancellation token source. - /// A task that represents the asynchronous operation. - public async ValueTask DisposeAsync() - { - _cts.Cancel(); - if (_pingLoop is not null) - { - try { await _pingLoop.ConfigureAwait(false); } catch { /* swallow shutdown errors */ } - } - _cts.Dispose(); - } -} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs index 56ac6ef7..6b0876d7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs @@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using ZB.MOM.WW.Auth.AspNetCore; using ZB.MOM.WW.Auth.Abstractions.Roles; using ZB.MOM.WW.OtOpcUa.Security.Jwt; @@ -36,7 +37,14 @@ public static class AuthEndpoints // AddOtOpcUaAuth fallback policy that would otherwise 401 the request. app.MapPost("/auth/login", (Delegate)LoginAsync).AllowAnonymous().DisableAntiforgery(); app.MapGet("/auth/ping", (Delegate)Ping).AllowAnonymous(); - app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(); + // Mapped only when a signing key is configured (JwtOptions.IsConfigured): an unconfigured + // endpoint used to answer HTTP 500 from the JwtTokenService constructor on every call. + if (app.ServiceProvider.GetRequiredService>().Value.IsConfigured) + app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(); + else + app.ServiceProvider.GetService()? + .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") + .LogInformation("Security:Jwt:SigningKey is not configured — /auth/token is not mapped and bearer tokens are not accepted"); app.MapPost("/auth/logout", (Delegate)LogoutAsync).RequireAuthorization(); return app; } @@ -73,8 +81,13 @@ public static class AuthEndpoints { result = await ldap.AuthenticateAsync(username, password, ct); } - catch (Exception) + catch (Exception ex) when (ex is not OperationCanceledException) { + // R03:2.7 — a directory outage used to leave NOTHING in the server log from this path; the + // operator had to infer every 503 from the directory side. + http.RequestServices.GetService()? + .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") + .LogWarning(ex, "LDAP authentication failed for {User}; answering 503", username); return Results.StatusCode(StatusCodes.Status503ServiceUnavailable); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs index 07cbdda4..f523a033 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs @@ -15,4 +15,26 @@ public sealed class JwtOptions /// Default token expiry. Mirrors ScadaLink (15 min). public int ExpiryMinutes { get; set; } = 15; + + /// + /// Whether the JWT feature is on: a non-blank . When off, /auth/token + /// is not mapped and no bearer scheme is registered, so an admin node that never configured a key + /// keeps booting exactly as before; when on, the key is validated at host start + /// (ValidateOnStart) instead of throwing HTTP 500 on the first token request. + /// + public bool IsConfigured => !string.IsNullOrWhiteSpace(SigningKey); + + /// The validation rule shared by the startup validator and the tests: why the options are invalid, or null. + /// A failure message, or when valid. + public string? Validate() + { + if (!IsConfigured) return null; + var keyBytes = System.Text.Encoding.UTF8.GetByteCount(SigningKey); + if (keyBytes < MinSigningKeyBytes) + return $"{SectionName}:SigningKey must be at least {MinSigningKeyBytes} bytes (256 bits) UTF-8 for HMAC-SHA256; the configured key is {keyBytes} byte(s)."; + if (string.IsNullOrWhiteSpace(Issuer)) return $"{SectionName}:Issuer must not be blank."; + if (string.IsNullOrWhiteSpace(Audience)) return $"{SectionName}:Audience must not be blank."; + if (ExpiryMinutes <= 0) return $"{SectionName}:ExpiryMinutes must be > 0."; + return null; + } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs new file mode 100644 index 00000000..c3805267 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.Options; + +namespace ZB.MOM.WW.OtOpcUa.Security.Jwt; + +/// +/// Startup validator for (R03:2.4): when a signing key is configured it must +/// be usable, and the host must learn that at start (ValidateOnStart) rather than from an HTTP 500 +/// on the first /auth/token. An unconfigured section (no key) is valid — the feature is simply off. +/// +public sealed class JwtOptionsValidator : IValidateOptions +{ + /// + public ValidateOptionsResult Validate(string? name, JwtOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var failure = options.Validate(); + return failure is null ? ValidateOptionsResult.Success : ValidateOptionsResult.Fail(failure); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs index 6df24cad..d58fbcf3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs @@ -19,9 +19,12 @@ using ZB.MOM.WW.OtOpcUa.Security.Ldap; namespace ZB.MOM.WW.OtOpcUa.Security; /// -/// DI registration for OtOpcUa auth. Single Cookie scheme (the JWT lives inside the -/// cookie as its credential payload); no JwtBearer parallel scheme. Matches ScadaBridge -/// structurally — see docs/plans/2026-05-29-auth-alignment-design.md. +/// DI registration for OtOpcUa auth. The cookie scheme carries the browser session; when +/// Security:Jwt:SigningKey is configured a scheme +/// accepts the tokens /auth/token mints (R03:2.4), and the default scheme is a policy scheme +/// that forwards to whichever the request presents — so /api/* and every +/// RequireAuthorization endpoint take a browser session OR a machine token. Matches +/// ScadaBridge structurally — see docs/plans/2026-05-29-auth-alignment-design.md. /// public static class ServiceCollectionExtensions { @@ -36,7 +39,14 @@ public static class ServiceCollectionExtensions /// The same service collection, for chaining. public static IServiceCollection AddOtOpcUaAuth(this IServiceCollection services, IConfiguration configuration) { - services.AddOptions().Bind(configuration.GetSection(JwtOptions.SectionName)); + // JWT is opt-in by key: with no SigningKey the feature is off (no /auth/token, no bearer scheme) + // and an admin node boots as before; with one, the key is validated at host START (R03:2.4) — + // the JwtTokenService constructor used to throw on the first /auth/token call (HTTP 500). + services.AddOptions() + .Bind(configuration.GetSection(JwtOptions.SectionName)) + .ValidateOnStart(); + services.AddSingleton, JwtOptionsValidator>(); + var jwtConfigured = configuration.GetSection(JwtOptions.SectionName).Get()?.IsConfigured ?? false; services.AddOptions().Bind(configuration.GetSection(OtOpcUaCookieOptions.SectionName)); services.AddOptions().Bind(configuration.GetSection(LdapOptions.SectionName)); services.AddOptions().Bind(configuration.GetSection(AuthDisableLoginOptions.SectionName)); @@ -53,7 +63,10 @@ public static class ServiceCollectionExtensions // as the Actor field when constructing a canonical ZB.MOM.WW.Audit.AuditEvent. Registered // Scoped so it correctly follows the request scope used by Blazor Server and minimal-API // endpoints. - services.TryAddScoped(); + // AuthenticationStateAuditActorAccessor reads the Blazor circuit's principal first (no HttpContext + // exists on a circuit's threads after the first render) and falls back to the HTTP context for + // minimal-API scopes, where AuthenticationStateProvider is not registered (R03:2.5). + services.TryAddScoped(); // Singleton — OtOpcUaLdapAuthService is stateless (the shared-library directory client it // wraps opens/disposes an LdapConnection per call) and must be consumable by the Singleton @@ -78,7 +91,26 @@ public static class ServiceCollectionExtensions .GetSection(AuthDisableLoginOptions.SectionName) .GetValue(nameof(AuthDisableLoginOptions.DisableLogin)); - var authBuilder = services.AddAuthentication(CookieAuthenticationDefaults.AuthenticationScheme); + // The default scheme is the cookie-or-bearer selector; sign-in/out stay on the cookie scheme, which is + // the only one that can issue a session. With JWT off the selector always picks the cookie. + var authBuilder = services.AddAuthentication(o => + { + o.DefaultScheme = OtOpcUaAuthSchemes.CookieOrBearer; + o.DefaultSignInScheme = CookieAuthenticationDefaults.AuthenticationScheme; + o.DefaultSignOutScheme = CookieAuthenticationDefaults.AuthenticationScheme; + }); + authBuilder.AddPolicyScheme(OtOpcUaAuthSchemes.CookieOrBearer, "Cookie or Bearer", o => + { + o.ForwardDefaultSelector = ctx => + jwtConfigured && OtOpcUaAuthSchemes.HasBearerToken(ctx) + ? OtOpcUaAuthSchemes.Bearer + : CookieAuthenticationDefaults.AuthenticationScheme; + }); + if (jwtConfigured) + { + authBuilder.AddScheme( + OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { }); + } if (disableLogin) { // DEV/TEST ONLY: replace the cookie handler with an always-succeeding handler registered @@ -142,8 +174,8 @@ public static class ServiceCollectionExtensions services.AddAuthorization(o => { - o.FallbackPolicy = new AuthorizationPolicyBuilder( - CookieAuthenticationDefaults.AuthenticationScheme) + // Named on the selector, not the cookie scheme, so the fallback accepts a bearer token too. + o.FallbackPolicy = new AuthorizationPolicyBuilder(OtOpcUaAuthSchemes.CookieOrBearer) .RequireAuthenticatedUser() .Build(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs new file mode 100644 index 00000000..f4e6fd22 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs @@ -0,0 +1,68 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:4.1 — admin operations echoed ex.Message to the UI, so a SqlException's connection-string +/// fragment reached any ConfigEditor browser session. The reply is now a fixed sentence plus the request's +/// correlation id; the detail stays in the central log. +/// +public sealed class AdminOperationsActorServerFaultMessageTests : ControlPlaneActorTestBase +{ + private const string Secret = "Server=sql-prod-01;Password=hunter2"; + + [Fact] + public void StartDeployment_server_fault_reply_carries_the_correlation_id_and_not_the_exception_text() + { + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(new ThrowingDbFactory(), coordinator.Ref, Enumerable.Empty())); + var correlationId = CorrelationId.NewId(); + + actor.Tell(new StartDeployment("joe", correlationId)); + + var reply = ExpectMsg(PresenceBudget); + reply.Outcome.ShouldBe(StartDeploymentOutcome.Rejected); + reply.Message.ShouldNotBeNull(); + reply.Message.ShouldNotContain("hunter2"); + reply.Message.ShouldNotContain("sql-prod-01"); + reply.Message.ShouldContain(correlationId.ToString()); // the operator can find the log line + } + + [Fact] + public void TestDriverConnect_probe_throw_reply_carries_the_correlation_id_and_not_the_exception_text() + { + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(NewInMemoryDbFactory(), coordinator.Ref, new IDriverProbe[] { new ThrowingProbe() })); + var correlationId = Guid.NewGuid(); + + actor.Tell(new TestDriverConnect("Throwing", "{}", 5, correlationId)); + + var reply = ExpectMsg(PresenceBudget); + reply.Ok.ShouldBeFalse(); + reply.Message.ShouldNotBeNull(); + reply.Message.ShouldNotContain("hunter2"); + reply.Message.ShouldContain(correlationId.ToString()); + } + + private sealed class ThrowingDbFactory : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => + throw new InvalidOperationException("Cannot open database: " + Secret); + } + + private sealed class ThrowingProbe : IDriverProbe + { + public string DriverType => "Throwing"; + public Task ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct) => + throw new InvalidOperationException("probe blew up: " + Secret); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs new file mode 100644 index 00000000..e91fbda0 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration; + +/// +/// R03:2.3 — an unset or placeholder pepper must fail host start, not surface as "Invalid API key" +/// on the first activation. +/// +public sealed class ApiKeyOptionsValidatorTests +{ + private static ValidateOptionsResult Validate(ApiKeyOptions options) => + new ApiKeyOptionsValidator().Validate(Options.DefaultName, options); + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Blank_pepper_fails(string pepper) + { + var result = Validate(new ApiKeyOptions { Pepper = pepper }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain("Pepper is required"); + } + + [Fact] + public void Short_pepper_fails() + { + var result = Validate(new ApiKeyOptions { Pepper = "x" }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain($"at least {ApiKeyOptionsValidator.MinPepperLength}"); + } + + [Fact] + public void Blank_username_fails() + { + var result = Validate(new ApiKeyOptions { Pepper = new string('p', 32), Username = " " }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain("Username"); + } + + /// Positive control: the docker-dev rig's pepper and the defaults pass unchanged. + [Fact] + public void Configured_pepper_passes() + { + Validate(new ApiKeyOptions { Pepper = "docker-dev-apikey-pepper-not-for-production" }) + .Succeeded.ShouldBeTrue(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs new file mode 100644 index 00000000..5dc1f391 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs @@ -0,0 +1,80 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Components.Authorization; +using Microsoft.AspNetCore.Http; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Auth.AspNetCore; +using ZB.MOM.WW.OtOpcUa.Security.Audit; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests.Audit; + +/// +/// R03:2.5 — inside a Blazor circuit there is no HttpContext after the first render, so an +/// HTTP-only accessor stamps every audit row from an interactive page "system". The circuit's +/// is consulted first; the HTTP context is the fallback. +/// +public sealed class AuthenticationStateAuditActorAccessorTests +{ + [Fact] + public void Circuit_principal_wins_when_there_is_no_http_context() + { + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new FixedStateProvider(Authenticated(new Claim(ZbClaimTypes.Username, "alice")))); + + accessor.CurrentActor.ShouldBe("alice"); + } + + [Fact] + public void Http_principal_is_the_fallback_outside_a_circuit() + { + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "bob")) }; + var accessor = new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), authenticationState: null); + + accessor.CurrentActor.ShouldBe("bob"); + } + + [Fact] + public void Unauthenticated_circuit_falls_through_to_http_then_null() + { + var anonymousCircuit = new FixedStateProvider(new ClaimsPrincipal(new ClaimsIdentity())); + + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(null), anonymousCircuit) + .CurrentActor.ShouldBeNull(); + + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Name, "carol")) }; + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), anonymousCircuit) + .CurrentActor.ShouldBe("carol"); + } + + /// An incomplete state task is "no circuit identity", never a block. + [Fact] + public void Pending_authentication_state_is_not_awaited() + { + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new PendingStateProvider()); + + accessor.CurrentActor.ShouldBeNull(); + } + + private static ClaimsPrincipal Authenticated(params Claim[] claims) => + new(new ClaimsIdentity(claims, "TestScheme", ZbClaimTypes.Name, ZbClaimTypes.Role)); + + private sealed class HttpContextAccessorStub(HttpContext? context) : IHttpContextAccessor + { + public HttpContext? HttpContext { get; set; } = context; + } + + private sealed class FixedStateProvider(ClaimsPrincipal user) : AuthenticationStateProvider + { + public override Task GetAuthenticationStateAsync() => + Task.FromResult(new AuthenticationState(user)); + } + + private sealed class PendingStateProvider : AuthenticationStateProvider + { + public override Task GetAuthenticationStateAsync() => + new TaskCompletionSource().Task; + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs index 9cf8ab11..e5e3b620 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs @@ -11,6 +11,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Shouldly; using Xunit; using ZB.MOM.WW.Auth.AspNetCore; @@ -37,6 +38,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime private IHost _host = null!; private TestServer _server = null!; private readonly StubLdapGroupRoleMappingService _roleMappings = new(); + private readonly CapturingLoggerProvider _logs = new(); private static CancellationToken Ct => TestContext.Current.CancellationToken; @@ -70,6 +72,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime ["Security:Ldap:GroupToRole:ReadOnly"] = "Viewer", }).Build(); services.AddOtOpcUaAuth(configuration); + services.AddLogging(l => l.AddProvider(_logs)); services.AddSingleton(); // The login handler now resolves the DB role-map service via DI to merge // DB-backed grants on top of the appsettings baseline. Register the stub so @@ -96,6 +99,10 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime .ToArray(); return Results.Ok(claims); }).RequireAuthorization(); + // A role-gated machine endpoint: proves a bearer principal's roles resolve through + // IsInRole (the validation parameters pair RoleClaimType with the minted key). + e.MapGet("/api/viewer-only", () => Results.Ok("viewer")) + .RequireAuthorization(p => p.RequireRole("Viewer")); }); }); }) @@ -153,6 +160,66 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime new AuthEndpoints.LoginRequest("ldap-down", "anything"), Ct); response.StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + // R03:2.7 — the outage is named in the server log, not inferred from the directory side. + _logs.Entries.Any(e => + e.Level == LogLevel.Warning && e.Message.Contains("LDAP authentication failed") && e.Exception != null) + .ShouldBeTrue("a Warning naming the LDAP failure must be logged"); + } + + /// + /// R03:2.4 — a token from /auth/token authenticates a request carrying it as a bearer: + /// the fallback-protected root AND a role-gated endpoint (the bearer principal's "Role" claims + /// resolve through IsInRole). No cookie is sent. + /// + [Fact] + public async Task Bearer_token_authenticates_api_requests_without_a_cookie() + { + var token = await LoginAndIssueTokenAsync(); + var client = NewClient(); + + var root = new HttpRequestMessage(HttpMethod.Get, "/"); + root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + (await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + var gated = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + gated.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + var gatedResponse = await client.SendAsync(gated, Ct); + gatedResponse.StatusCode.ShouldBe(HttpStatusCode.OK); + (await gatedResponse.Content.ReadAsStringAsync(Ct)).ShouldContain("viewer"); + } + + /// A tampered or garbage bearer fails closed (401), it never falls through to anonymous. + [Theory] + [InlineData("not-a-jwt")] + [InlineData("tampered")] + public async Task Bad_bearer_token_is_rejected(string kind) + { + var token = kind == "tampered" + ? (await LoginAndIssueTokenAsync()) + "x" + : kind; + var client = NewClient(); + var req = new HttpRequestMessage(HttpMethod.Get, "/"); + req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + req.Headers.Add("X-Requested-With", "XMLHttpRequest"); + + var response = await client.SendAsync(req, Ct); + + response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized); + response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); + } + + private async Task LoginAndIssueTokenAsync() + { + var client = NewClient(); + var loginResponse = await client.PostAsJsonAsync("/auth/login", + new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + loginResponse.EnsureSuccessStatusCode(); + var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + AttachCookies(tokenReq, loginResponse); + var response = await client.SendAsync(tokenReq, Ct); + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadFromJsonAsync(Ct); + return payload.GetProperty("token").GetString()!; } /// Tests that ping anonymous returns 401. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs new file mode 100644 index 00000000..a79ab569 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs @@ -0,0 +1,25 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests; + +/// Collects every log entry written through the host's logger factory, for assertions on what was logged. +internal sealed class CapturingLoggerProvider : ILoggerProvider +{ + public ConcurrentQueue Entries { get; } = new(); + + public ILogger CreateLogger(string categoryName) => new Logger(categoryName, Entries); + + public void Dispose() { } + + internal sealed record Entry(string Category, LogLevel Level, string Message, Exception? Exception); + + private sealed class Logger(string category, ConcurrentQueue entries) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + entries.Enqueue(new Entry(category, logLevel, formatter(state, exception), exception)); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs new file mode 100644 index 00000000..96fc88ca --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs @@ -0,0 +1,127 @@ +using System.Net; +using System.Net.Http.Json; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Services; +using ZB.MOM.WW.OtOpcUa.Security.Auth; +using ZB.MOM.WW.OtOpcUa.Security.Endpoints; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; +using ZB.MOM.WW.OtOpcUa.Security.Ldap; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests; + +/// +/// R03:2.4 — JWT is opt-in by signing key. Unconfigured: /auth/token is not mapped (it used to +/// answer HTTP 500 from the service constructor) and no bearer scheme exists. Configured badly: the +/// host refuses to START rather than failing on the first token request. +/// +public sealed class JwtConfigurationGateTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Without_a_signing_key_auth_token_is_not_mapped_and_bearer_is_ignored() + { + using var host = await StartHostAsync(signingKey: null); + var server = host.GetTestServer(); + var client = server.CreateClient(); + + // The login still works — the cookie path is untouched. + var login = await client.PostAsJsonAsync("/auth/login", new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + login.StatusCode.ShouldBe(HttpStatusCode.NoContent); + + // /auth/token is simply absent (404), not a 500. + var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + foreach (var c in login.Headers.GetValues("Set-Cookie")) tokenReq.Headers.Add("Cookie", c.Split(';')[0]); + (await client.SendAsync(tokenReq, Ct)).StatusCode.ShouldBe(HttpStatusCode.NotFound); + + // No bearer scheme is registered, so a bearer header is ignored and the cookie session still wins. + var schemes = host.Services.GetRequiredService(); + (await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer)).ShouldBeNull(); + var withBogusBearer = new HttpRequestMessage(HttpMethod.Get, "/"); + foreach (var c in login.Headers.GetValues("Set-Cookie")) withBogusBearer.Headers.Add("Cookie", c.Split(';')[0]); + withBogusBearer.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "garbage"); + (await client.SendAsync(withBogusBearer, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + await host.StopAsync(Ct); + } + + /// Positive control for the gate: with a key, the bearer scheme IS registered. + [Fact] + public async Task With_a_signing_key_the_bearer_scheme_is_registered() + { + using var host = await StartHostAsync(signingKey: "test-signing-key-with-at-least-32-bytes-of-utf8-content"); + var schemes = host.Services.GetRequiredService(); + var bearer = await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer); + bearer.ShouldNotBeNull(); + bearer.HandlerType.ShouldBe(typeof(JwtBearerAuthenticationHandler)); + await host.StopAsync(Ct); + } + + /// A configured-but-short key fails at host start (ValidateOnStart), not on the first /auth/token. + [Fact] + public async Task A_short_signing_key_fails_host_start() + { + var ex = await Should.ThrowAsync(async () => + { + using var host = await StartHostAsync(signingKey: "too-short"); + }); + ex.Message.ShouldContain("SigningKey"); + } + + private static async Task StartHostAsync(string? signingKey) + { + var dbName = $"jwt-gate-{Guid.NewGuid():N}"; + var settings = new Dictionary(); + if (signingKey is not null) settings["Security:Jwt:SigningKey"] = signingKey; + var configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build(); + + var host = new HostBuilder() + .ConfigureWebHost(web => + { + web.UseTestServer(); + web.ConfigureServices(services => + { + services.AddDbContextFactory(opt => opt.UseInMemoryDatabase(dbName)); + services.AddDbContext(opt => opt.UseInMemoryDatabase(dbName)); + services.AddRouting(); + services.AddOtOpcUaAuth(configuration); + services.AddSingleton(); + services.AddSingleton(); + }); + web.Configure(app => + { + app.UseRouting(); + app.UseAuthentication(); + app.UseAuthorization(); + app.UseEndpoints(e => + { + e.MapOtOpcUaAuth(); + e.MapGet("/", () => Results.Ok("authenticated")).RequireAuthorization(); + }); + }); + }) + .Build(); + await host.StartAsync(Ct); + return host; + } + + private sealed class AlwaysAliceLdap : ILdapAuthService + { + public Task AuthenticateAsync(string username, string password, CancellationToken ct = default) => + Task.FromResult(new LdapAuthResult( + Success: password == "valid-password", DisplayName: "Alice", Username: username, + Groups: [], Roles: [], Error: null)); + } +} From 1bf5ef916cedd6ab532edca25e6492320df048e3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:27:59 -0400 Subject: [PATCH 06/19] feat(controlplane): deployment artifact retention on the admin singleton MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every deploy stored the whole gzip'd fleet snapshot in Deployment.ArtifactBlob forever (R03:1.2). DeploymentRetentionActor (admin singleton, proxy-less) sweeps every Deployments:Retention: IntervalMinutes (60): terminal rows (Sealed / PartiallyFailed / TimedOut) older than the newest KeepSealed (50) and not referenced by a NodeDeploymentState started/applied within ReferencedDays (30) get their blob EMPTIED — never deleted, never NULLed: the column stays NOT NULL and every reader already treats zero bytes as "no artifact" (#485), so no schema migration and no reader change. In-flight rows are never touched; MaxRowsPerSweep (200) bounds one sweep. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Entities/Deployment.cs | 6 +- .../Retention/DeploymentRetentionActor.cs | 177 ++++++++++++++++++ .../Retention/DeploymentRetentionOptions.cs | 54 ++++++ .../ServiceCollectionExtensions.cs | 18 ++ .../DeploymentRetentionTests.cs | 174 +++++++++++++++++ 5 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs index 19d83728..d3044244 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs @@ -26,7 +26,11 @@ public sealed class Deployment public DateTime CreatedAtUtc { get; init; } = DateTime.UtcNow; /// Gets or sets the serialized artifact blob containing the configuration. - public byte[] ArtifactBlob { get; init; } = Array.Empty(); + /// + /// Settable (not init-only) so the retention sweep can EMPTY it on an old terminal row. An empty blob + /// is "no artifact" to every reader (#485), so the audit row survives with its bytes retired. + /// + public byte[] ArtifactBlob { get; set; } = Array.Empty(); /// Gets or sets the row version for optimistic concurrency control. public byte[] RowVersion { get; set; } = Array.Empty(); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs new file mode 100644 index 00000000..55ffd7dc --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs @@ -0,0 +1,177 @@ +using Akka.Actor; +using Akka.Event; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; + +/// +/// Admin-role cluster singleton that periodically empties the ArtifactBlob of old, terminal +/// deployments (R03:1.2). See for the policy and +/// for the rule, which is pure over a context so it is unit-testable. +/// +/// +/// +/// Empties, never deletes, and never nulls. The column is NOT NULL varbinary(max), and +/// every reader (DriverHostActor.LoadArtifactBlob, OpcUaPublishActor, +/// DeploymentArtifactService) already treats a zero-length blob as "no artifact" (#485: empty +/// bytes are not an empty configuration — the apply fails and the node keeps last-known-good). An +/// empty blob therefore needs no schema change and no reader change, where a NULL would need both. +/// +/// +/// Never touches an in-flight row (Dispatching / AwaitingApplyAcks): a node may still be +/// fetching it. The newest terminal rows keep +/// their blob whatever their age, so the "latest sealed" every boot path reads is always intact. +/// +/// +public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers +{ + private readonly IDbContextFactory _dbFactory; + private readonly DeploymentRetentionOptions _options; + private readonly ILoggingAdapter _log = Context.GetLogger(); + private const string SweepTimerKey = "retention-sweep"; + + /// Gets or sets the timer scheduler. + public ITimerScheduler Timers { get; set; } = null!; + + /// Creates the actor props. + /// The config DB factory. + /// The retention policy. + /// The props. + public static Props Props(IDbContextFactory dbFactory, DeploymentRetentionOptions options) => + Akka.Actor.Props.Create(() => new DeploymentRetentionActor(dbFactory, options)); + + /// Initializes a new instance of the class. + /// The config DB factory. + /// The retention policy. + public DeploymentRetentionActor(IDbContextFactory dbFactory, DeploymentRetentionOptions options) + { + ArgumentNullException.ThrowIfNull(dbFactory); + ArgumentNullException.ThrowIfNull(options); + _dbFactory = dbFactory; + _options = options; + + Receive(_ => RunSweep()); + Receive(msg => + { + if (msg.Pruned > 0) + _log.Info("Deployment retention emptied the artifact blob on {Count} old deployment(s)", msg.Pruned); + else + _log.Debug("Deployment retention sweep found nothing to prune"); + if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg); + }); + Receive(msg => + { + _log.Warning(msg.Error, "Deployment retention sweep failed; the next sweep retries"); + if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg); + }); + } + + /// + protected override void PreStart() + { + if (!_options.Enabled) + { + _log.Info("Deployment retention is disabled (Deployments:Retention:Enabled=false); artifact blobs are kept forever"); + return; + } + var interval = TimeSpan.FromMinutes(Math.Max(1, _options.IntervalMinutes)); + var initial = TimeSpan.FromSeconds(Math.Max(0, _options.InitialDelaySeconds)); + Timers.StartPeriodicTimer(SweepTimerKey, Sweep.Instance, initial, interval); + } + + private void RunSweep() + { + var replyTo = Sender.IsNobody() ? null : Sender; + var self = Self; + var now = DateTime.UtcNow; + Task.Run(async () => + { + await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false); + return await SweepAsync(db, _options, now).ConfigureAwait(false); + }) + .PipeTo(self, + success: pruned => new SweepCompleted(pruned, replyTo), + failure: ex => new SweepFailed(ex, replyTo)); + } + + /// + /// The retention rule. Empties the blob of every terminal deployment that is (a) older than the + /// newest terminal rows, (b) not referenced by a + /// NodeDeploymentState row started or applied within + /// , and (c) still carrying bytes — at most + /// per call. + /// + /// The config DB. + /// The policy. + /// The sweep's clock. + /// Cancellation. + /// How many rows were emptied. + public static async Task SweepAsync( + OtOpcUaConfigDbContext db, DeploymentRetentionOptions options, DateTime utcNow, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(options); + + var keep = Math.Max(1, options.KeepSealed); + var referenceFloor = utcNow.AddDays(-Math.Max(0, options.ReferencedDays)); + var batch = Math.Max(1, options.MaxRowsPerSweep); + + // Projected: ids only. The candidate scan must not materialise blobs for rows it will keep. + var terminal = await db.Deployments + .AsNoTracking() + .Where(d => d.Status == DeploymentStatus.Sealed + || d.Status == DeploymentStatus.PartiallyFailed + || d.Status == DeploymentStatus.TimedOut) + .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, HasBlob = d.ArtifactBlob.Length > 0 }) + .ToListAsync(ct) + .ConfigureAwait(false); + + var aged = terminal.Skip(keep).Where(d => d.HasBlob).Select(d => d.DeploymentId).ToList(); + if (aged.Count == 0) return 0; + + var referenced = (await db.NodeDeploymentStates + .AsNoTracking() + .Where(s => aged.Contains(s.DeploymentId) + && (s.StartedAtUtc >= referenceFloor + || (s.AppliedAtUtc != null && s.AppliedAtUtc >= referenceFloor))) + .Select(s => s.DeploymentId) + .Distinct() + .ToListAsync(ct) + .ConfigureAwait(false)) + .ToHashSet(); + + var targets = aged.Where(id => !referenced.Contains(id)).Take(batch).ToList(); + if (targets.Count == 0) return 0; + + // Tracked load of ONLY the rows being emptied: the update rides the row's real RowVersion on + // both providers, and the blob read is the one-time cost of retiring that row. + var rows = await db.Deployments + .Where(d => targets.Contains(d.DeploymentId)) + .ToListAsync(ct) + .ConfigureAwait(false); + foreach (var row in rows) row.ArtifactBlob = []; + await db.SaveChangesAsync(ct).ConfigureAwait(false); + return rows.Count; + } + + /// Asks for a sweep now; the sender (if any) receives the / . + public sealed class Sweep + { + /// The singleton instance. + public static readonly Sweep Instance = new(); + private Sweep() { } + } + + /// A sweep finished. + /// How many rows were emptied. + /// Who asked, when the sweep was requested rather than timer-driven. + public sealed record SweepCompleted(int Pruned, IActorRef? ReplyTo); + + /// A sweep threw. + /// The failure. + /// Who asked, when the sweep was requested rather than timer-driven. + public sealed record SweepFailed(Exception Error, IActorRef? ReplyTo); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs new file mode 100644 index 00000000..6654ec1e --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; + +/// +/// Deployments:Retention — how long a sealed deployment keeps its ArtifactBlob (R03:1.2). +/// Every deploy stored the whole gzip'd fleet snapshot forever; nothing pruned it, so a year of routine +/// edits was gigabytes of blobs. The retention sweep EMPTIES the blob on old rows and never deletes a +/// row: the audit record (who deployed what revision, when, and how every node acked) survives. +/// +public sealed class DeploymentRetentionOptions +{ + /// The configuration section name. + public const string SectionName = "Deployments:Retention"; + + /// Master switch; keeps every blob forever (the previous behaviour). + public bool Enabled { get; set; } = true; + + /// + /// How many of the most recent terminal (Sealed / PartiallyFailed / TimedOut) deployments keep their + /// blob regardless of age. A node booting from central SQL reads the LATEST sealed row's blob, so this + /// must stay well above 1; 50 also keeps a comfortable re-deploy history. + /// + public int KeepSealed { get; set; } = 50; + + /// + /// A deployment any NodeDeploymentState row started or applied within this many days keeps its + /// blob even when older than the window — a node that just applied it may + /// still replay it on restart. + /// + public int ReferencedDays { get; set; } = 30; + + /// How often the sweep runs. + public int IntervalMinutes { get; set; } = 60; + + /// Delay before the first sweep after the singleton starts, so a boot is not a blob scan. + public int InitialDelaySeconds { get; set; } = 60; + + /// + /// Rows emptied per sweep. The sweep loads each candidate row (blob included) to update it under its + /// RowVersion, so this bounds one sweep's memory and transaction size; a backlog drains over sweeps. + /// + public int MaxRowsPerSweep { get; set; } = 200; + + /// Binds the section from configuration, with defaults for anything absent. + /// The app configuration; null yields the defaults. + /// The bound options. + public static DeploymentRetentionOptions Bind(IConfiguration? configuration) + { + var options = new DeploymentRetentionOptions(); + configuration?.GetSection(SectionName).Bind(options); + return options; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index cb8d605a..f838a004 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -15,6 +15,7 @@ using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.ControlPlane; @@ -30,6 +31,7 @@ public static class ServiceCollectionExtensions public const string AuditWriterSingletonName = "audit-writer"; public const string RedundancyStateSingletonName = "redundancy-state"; public const string ClusterNodeAddressReconcilerSingletonName = "cluster-node-address-reconciler"; + public const string DeploymentRetentionSingletonName = "deployment-retention"; /// /// Registers the admin-role cluster singletons on the AkkaConfigurationBuilder — three with @@ -181,6 +183,21 @@ public static class ServiceCollectionExtensions singletonOptions, createProxyToo: false); + // Deployment artifact retention (R03:1.2): empties ArtifactBlob on old terminal deployments so the + // Deployment table stops growing by one fleet snapshot per edit. Admin singleton for the same + // reason as the reconciler (one sweeper per fleet, and only central holds the ConfigDb); + // proxy-less because nothing addresses it in production — a test sends Sweep directly. + builder.WithSingleton( + DeploymentRetentionSingletonName, + (system, registry, resolver) => + { + var dbFactory = resolver.GetService>(); + var options = DeploymentRetentionOptions.Bind(resolver.GetService()); + return DeploymentRetentionActor.Props(dbFactory, options); + }, + singletonOptions, + createProxyToo: false); + return builder; } @@ -256,4 +273,5 @@ public sealed class AdminOperationsActorKey { } public sealed class AuditWriterActorKey { } public sealed class RedundancyStateActorKey { } public sealed class ClusterNodeAddressReconcilerKey { } +public sealed class DeploymentRetentionKey { } public sealed class CentralCommunicationActorKey { } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs new file mode 100644 index 00000000..ecd397c4 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs @@ -0,0 +1,174 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:1.2 — every deploy stored the whole fleet snapshot forever. The retention sweep empties the +/// blob on old terminal rows (never deletes a row, never touches an in-flight one, never the newest N, +/// never one a node applied recently). +/// +public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase +{ + private static readonly DateTime SweepNow = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc); + + [Fact] + public async Task Sweep_empties_blobs_beyond_the_keep_window_and_leaves_the_newest_intact() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 12); // ids[0] newest … ids[11] oldest + var options = new DeploymentRetentionOptions { KeepSealed = 5, ReferencedDays = 30, MaxRowsPerSweep = 200 }; + + int pruned; + await using (var db = f.CreateDbContext()) + pruned = await DeploymentRetentionActor.SweepAsync(db, options, SweepNow); + + pruned.ShouldBe(7); + await using var verify = f.CreateDbContext(); + foreach (var id in ids.Take(5)) + (await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.Length.ShouldBeGreaterThan(0, "newest 5 keep their blob"); + foreach (var id in ids.Skip(5)) + (await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.ShouldBeEmpty("older rows are emptied"); + // Rows survive: the audit record is intact. + (await verify.Deployments.CountAsync()).ShouldBe(12); + + // Idempotent: a second sweep finds nothing. + await using var again = f.CreateDbContext(); + (await DeploymentRetentionActor.SweepAsync(again, options, SweepNow)).ShouldBe(0); + } + + [Fact] + public async Task Sweep_keeps_a_row_a_node_applied_recently_and_never_touches_an_in_flight_row() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 4); + var inFlight = Guid.NewGuid(); + await using (var db = f.CreateDbContext()) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1", + ApplicationUri = "urn:x", Enabled = true, CreatedBy = "test", + }); + // ids[3] is the oldest; a node applied it yesterday, so it stays. + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = "site-a-1:4053", DeploymentId = ids[3], Status = NodeDeploymentStatus.Applied, + StartedAtUtc = SweepNow.AddDays(-1), AppliedAtUtc = SweepNow.AddDays(-1), + }); + // ids[2]: applied long ago — not protected. + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = "site-a-1:4053", DeploymentId = ids[2], Status = NodeDeploymentStatus.Applied, + StartedAtUtc = SweepNow.AddDays(-90), AppliedAtUtc = SweepNow.AddDays(-90), + }); + // An ancient in-flight row with a blob — never touched. + db.Deployments.Add(new Deployment + { + DeploymentId = inFlight, RevisionHash = new string('f', 64), Status = DeploymentStatus.AwaitingApplyAcks, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-400), ArtifactBlob = new byte[] { 1, 2, 3 }, + }); + await db.SaveChangesAsync(); + } + var options = new DeploymentRetentionOptions { KeepSealed = 1, ReferencedDays = 30 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2); // ids[1], ids[2] + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty("newest"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[1])).ArtifactBlob.ShouldBeEmpty(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[2])).ArtifactBlob.ShouldBeEmpty("applied 90 days ago"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[3])).ArtifactBlob.ShouldNotBeEmpty("applied yesterday"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == inFlight)).ArtifactBlob.ShouldNotBeEmpty("in flight"); + } + + [Fact] + public async Task Sweep_is_bounded_per_call_and_drains_a_backlog_over_calls() + { + var f = NewInMemoryDbFactory(); + Seed(f, sealedCount: 10); + var options = new DeploymentRetentionOptions { KeepSealed = 2, MaxRowsPerSweep = 3 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(0); + } + + /// The actor runs the same rule off its thread and reports back. + [Fact] + public void Actor_sweeps_on_request_and_reports_the_count() + { + var f = NewInMemoryDbFactory(); + Seed(f, sealedCount: 6); + var actor = Sys.ActorOf(DeploymentRetentionActor.Props(f, new DeploymentRetentionOptions + { + KeepSealed = 4, InitialDelaySeconds = 3600, IntervalMinutes = 60, + })); + + actor.Tell(DeploymentRetentionActor.Sweep.Instance); + + ExpectMsg(PresenceBudget).Pruned.ShouldBe(2); + using var verify = f.CreateDbContext(); + verify.Deployments.Count(d => d.ArtifactBlob.Length == 0).ShouldBe(2); + } + + [Fact] + public void Options_bind_from_configuration_with_defaults() + { + var defaults = DeploymentRetentionOptions.Bind(null); + defaults.Enabled.ShouldBeTrue(); + defaults.KeepSealed.ShouldBe(50); + defaults.ReferencedDays.ShouldBe(30); + defaults.IntervalMinutes.ShouldBe(60); + + var config = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Deployments:Retention:KeepSealed"] = "7", + ["Deployments:Retention:Enabled"] = "false", + }) + .Build(); + var bound = DeploymentRetentionOptions.Bind(config); + bound.KeepSealed.ShouldBe(7); + bound.Enabled.ShouldBeFalse(); + bound.ReferencedDays.ShouldBe(30); + } + + /// Seeds sealed deployments, newest first; each one hour older than the previous. + private static List Seed(IDbContextFactory f, int sealedCount) + { + var ids = new List(); + using var db = f.CreateDbContext(); + for (var i = 0; i < sealedCount; i++) + { + var id = Guid.NewGuid(); + ids.Add(id); + db.Deployments.Add(new Deployment + { + DeploymentId = id, + RevisionHash = new string('a', 64), + Status = i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed, + CreatedBy = "test", + CreatedAtUtc = SweepNow.AddHours(-i), + SealedAtUtc = SweepNow.AddHours(-i), + ArtifactBlob = new byte[] { 0x1f, 0x8b, (byte)i }, + }); + } + db.SaveChanges(); + return ids; + } +} From 18352442ab93f207acba2986a0c310968490578c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:30:03 -0400 Subject: [PATCH 07/19] fix(controlplane): audit writer drops only the rejected row, not the whole batch AuditWriterActor documented per-row dedup on the UX_ConfigAuditLog_EventId index but flushed one statement batch: a single duplicate EventId (any at-least-once producer across a singleton hand-over) threw for all 500 rows (R03:2.6). SaveChanges now detaches the entries a DbUpdateException names (2601/2627 logged at Debug as the expected duplicate, anything else at Warning) and re-saves, bounded by the batch size; a failure naming no entry stays a whole-batch failure. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Audit/AuditWriterActor.cs | 57 ++++++++- .../AuditWriterActorRejectedRowTests.cs | 110 ++++++++++++++++++ 2 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs index 2ca3026c..0e20da2f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs @@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; /// Dedup is two-layer: in-buffer (the below collapses /// duplicate EventIds before flush) and at the database via the filtered unique index /// UX_ConfigAuditLog_EventId (cross-restart safety — a retry of an already-flushed -/// batch hits the constraint and we drop the duplicate insert without losing the rest of -/// the batch). +/// batch hits the constraint; detaches the rejected row +/// and re-saves, so the duplicate is dropped without losing the rest of the batch). /// /// Implements the shared seam: is a /// best-effort, never-throwing entry point that simply Tells this actor and returns @@ -107,8 +107,8 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter Outcome = evt.Outcome.ToString(), }); } - db.SaveChanges(); - _log.Debug("AuditWriter flushed {Count} events", snapshot.Count); + var dropped = SaveDetachingRejectedRows(db); + _log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped); } catch (Exception ex) { @@ -116,6 +116,55 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter } } + /// + /// Saves the batch, detaching and retrying past any row the database rejects (R03:2.6). One + /// statement batch used to carry the whole flush: a single duplicate EventId (any at-least-once + /// producer — the audit seam is Tell-based through a singleton proxy that re-delivers across a + /// hand-over) threw for all 500 rows and the class remarks' per-row dedup claim was not what ran. + /// + /// The context holding the batch. + /// How many rows were dropped. + /// + /// Only a that NAMES its entries is handled here — that is the shape a + /// constraint violation takes (SQL 2601/2627 for the unique index). A failure with no entries (the + /// connection died, a deadlock) is the whole-batch failure it always was and propagates to the + /// caller's catch. Bounded by the batch size so a store that rejects every row still terminates. + /// + private int SaveDetachingRejectedRows(OtOpcUaConfigDbContext db) + { + var dropped = 0; + var attempts = 0; + var maxAttempts = db.ChangeTracker.Entries().Count() + 1; + while (true) + { + try + { + db.SaveChanges(); + return dropped; + } + catch (DbUpdateException ex) when (ex.Entries.Count > 0 && ++attempts < maxAttempts) + { + var duplicate = IsUniqueViolation(ex); + foreach (var entry in ex.Entries) + { + var eventId = (entry.Entity as ConfigAuditLog)?.EventId; + if (duplicate) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", eventId); + else + _log.Warning(ex, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", eventId); + entry.State = EntityState.Detached; + dropped++; + } + } + } + } + + /// SQL Server 2601 (unique index) / 2627 (unique constraint), the dedup contract's expected rejection. + /// The update failure. + /// for a duplicate-key rejection. + private static bool IsUniqueViolation(DbUpdateException ex) => + ex.InnerException is Microsoft.Data.SqlClient.SqlException { Number: 2601 or 2627 }; + /// protected override void PreRestart(Exception reason, object message) { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs new file mode 100644 index 00000000..bd64161a --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs @@ -0,0 +1,110 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.EntityFrameworkCore.Update; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:2.6 — one constraint hit used to drop the whole flush. The writer now detaches the rejected +/// row(s) and re-saves, so a duplicate EventId costs exactly that row. +/// +public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase +{ + [Fact] + public void A_rejected_row_is_dropped_and_the_rest_of_the_batch_is_persisted() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var duplicate = Guid.NewGuid(); + var rejecting = new RejectingDbFactory(dbName, root, new RejectOnce(duplicate)); + var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + + Watch(actor); + actor.Tell(PoisonPill.Instance); // PostStop flushes + ExpectTerminated(actor); + + using var db = rejecting.CreateDbContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(20, "every row but the rejected one lands"); + persisted.ShouldNotContain(duplicate); + // Positive control: the rows on either side of the rejected one are present. + persisted.ShouldContain(ids[6]); + persisted.ShouldContain(ids[8]); + } + + /// A failure that names no entry is a whole-batch failure, exactly as before. + [Fact] + public void A_connection_level_failure_still_drops_the_batch_without_looping() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var rejecting = new RejectingDbFactory(dbName, root, new AlwaysFailWithoutEntries()); + var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); + + for (var i = 0; i < 5; i++) actor.Tell(NewEvent(Guid.NewGuid())); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor); // terminates ⇒ the retry loop did not spin + + using var db = rejecting.CreateDbContext(); + db.ConfigAuditLogs.Count().ShouldBe(0); + } + + private static AuditEvent NewEvent(Guid eventId) => new() + { + EventId = eventId, Category = "Config", Action = "Edit", Actor = "joe", + OccurredAtUtc = DateTimeOffset.UtcNow, Outcome = AuditOutcome.Success, + DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(), + }; + + /// Throws a naming the entry for one EventId, the first time it is saved. + private sealed class RejectOnce(Guid eventId) : ISaveChangesInterceptor + { + private int _fired; + + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) + { + if (_fired == 1 || eventData.Context is null) return result; + var entry = eventData.Context.ChangeTracker.Entries() + .FirstOrDefault(e => e.Entity.EventId == eventId); + if (entry is null) return result; + _fired = 1; + // GetInfrastructure() hands back the pubternal InternalEntityEntry, which is the IUpdateEntry the + // real provider puts on the exception; the test only needs that one cast. +#pragma warning disable EF1001 + var updateEntry = (IUpdateEntry)entry.GetInfrastructure(); +#pragma warning restore EF1001 + throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601)", new[] { updateEntry }); + } + } + + private sealed class AlwaysFailWithoutEntries : ISaveChangesInterceptor + { + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) => + throw new DbUpdateException("connection lost (simulated)"); + } + + private sealed class RejectingDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor interceptor) + : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => new( + new DbContextOptionsBuilder() + .UseInMemoryDatabase(dbName, root) + .AddInterceptors(interceptor) + .Options); + } +} From 63714e430a75dadb1aa48c4a6106d6aec5165b48 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:34:39 -0400 Subject: [PATCH 08/19] fix(controlplane): evict retired mesh channels and refresh PSKs off the request path - MeshGrpcCommandDialer.RetainOnly (IMeshCommandDialer, default no-op): CentralCommunicationActor hands the dialer the current endpoint set on every contact refresh, so a renamed host or moved port releases its cached GrpcChannel and idle HTTP/2 connection instead of living for the process (R03:1.3). - IMeshPskProvider.RefreshAsync (default = the old InvalidateAll): MeshPskProvider pre-resolves every known cluster's key concurrently, bounded like the sync path, and swaps an immutable snapshot in atomically; a failed read keeps that cluster's previous key. The actor calls it from ContactsLoaded (after SetKnownClusters, off-actor via PipeTo) instead of InvalidateAll, so the first ack per cluster per ContactRefreshSeconds no longer blocks a gRPC thread on a 3 s-bounded store read or gets denied past the bound (R03:3.4). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../CentralCommunicationActor.cs | 57 +++++++- .../Communication/IMeshPskProvider.cs | 117 +++++++++++++++- .../Communication/MeshGrpcCommandDialer.cs | 33 +++++ .../CentralCommunicationActorTests.cs | 41 ++++++ .../MeshGrpcCommandDialerTests.cs | 29 ++++ .../MeshPskProviderRefreshTests.cs | 129 ++++++++++++++++++ 6 files changed, 404 insertions(+), 2 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs index cb1ea253..16f7b5d4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs @@ -201,6 +201,9 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers LoadContactsFromDb(); }); Receive(HandleContactsLoaded); + Receive(msg => _log.Debug("Mesh PSK refresh completed for {Count} cluster(s)", msg.ClusterCount)); + Receive(msg => _log.Warning(msg.Error, + "Mesh PSK refresh failed; previously resolved keys stay in service until the next refresh")); Receive(HandleCommandDispatched); // A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler @@ -710,6 +713,20 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers _targets = msg.Targets; + // R03:1.3 — release the channels of endpoints that left the dispatch set (host renamed, port + // moved). Only the resolved dialer is asked; nothing is resolved just to prune it. + if (_dialer is { } dialer) + { + try + { + dialer.RetainOnly(_targets.Select(t => t.Endpoint).ToHashSet(StringComparer.Ordinal)); + } + catch (Exception ex) + { + _log.Warning(ex, "Mesh dialer failed to release channels for retired endpoints; they stay cached until the next refresh"); + } + } + if (_targets.Count == 0) { _log.Warning( @@ -780,7 +797,37 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers } pskProvider.SetKnownClusters(msg.ClusterIds); - pskProvider.InvalidateAll(); + + // R03:3.4 — refresh the per-cluster keys OFF the request path and swap them in atomically, + // instead of dropping the cache and making the first ack per cluster re-read the store on a + // gRPC thread. A provider that does not pre-resolve (the interface default, and every fake) + // still drops its cache here, synchronously, exactly as before. + CancellationToken ct; + try + { + ct = _lifecycle.Token; + } + catch (ObjectDisposedException) + { + return; // Stopping. + } + var self = Self; + Task refresh; + try + { + refresh = pskProvider.RefreshAsync(msg.ClusterIds, ct); + } + catch (Exception ex) + { + _log.Warning(ex, "Mesh PSK refresh threw synchronously; cached keys are unchanged until the next refresh"); + refresh = Task.CompletedTask; + } + if (!refresh.IsCompletedSuccessfully) + { + refresh.PipeTo(self, + success: () => new PskRefreshed(msg.ClusterIds.Count), + failure: ex => new PskRefreshFailed(ex)); + } if (!_gatePublished) { @@ -795,6 +842,14 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// Internal tick asking for a contact-point refresh. public sealed record RefreshContacts; + /// The asynchronous PSK refresh (R03:3.4) completed. + /// How many clusters were refreshed. + public sealed record PskRefreshed(int ClusterCount); + + /// The asynchronous PSK refresh (R03:3.4) threw; the previous keys stay in service. + /// The failure. + public sealed record PskRefreshFailed(Exception Error); + /// Result of a contact-point load, piped back from the DB read. /// /// The gRPC dispatch set: one entry per enabled, non-maintenance row that carries a diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs index a7282694..bdab7ba7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs @@ -106,6 +106,26 @@ public interface IMeshPskProvider /// Drops every cached key. void InvalidateAll(); + + /// + /// Re-reads the keys for from the store OFF the request path and + /// swaps them into the cache atomically (R03:3.4). Until the refresh completes, lookups keep + /// serving the previous key; a cluster whose read fails keeps its previous key too. Replaces + /// "invalidate on every refresh", which made the first inbound ack per cluster per + /// ContactRefreshSeconds block a gRPC thread on a synchronous, 3 s-bounded store read. + /// + /// The clusters central currently knows (the same set given to ). + /// Cancels the refresh. + /// Completes when the swap has happened. + /// + /// The default implementation is the pre-R03:3.4 behaviour — drop the cache and let the next lookup + /// re-read — so a fake or a non-caching provider needs nothing more. + /// + Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) + { + InvalidateAll(); + return Task.CompletedTask; + } } /// @@ -144,6 +164,16 @@ public sealed class MeshPskProvider : IMeshPskProvider private readonly ConcurrentDictionary> _cache = new(StringComparer.OrdinalIgnoreCase); + /// + /// Keys pre-resolved by , consulted after the config map and before the + /// lazy . Swapped wholesale (volatile reference to an immutable dictionary), + /// never mutated, so a lookup racing a refresh sees the whole previous snapshot or the whole next + /// one. A cluster missing from it — never refreshed, or whose read failed — falls through to the + /// lazy path exactly as before, which is what keeps "refresh failed" from meaning "no key". + /// + private volatile IReadOnlyDictionary _refreshed = + new Dictionary(StringComparer.OrdinalIgnoreCase); + /// /// The known-cluster gate's membership set, or when the gate is /// inactive (never published, or published empty — see ). @@ -210,6 +240,13 @@ public sealed class MeshPskProvider : IMeshPskProvider return true; } + // Pre-resolved by the asynchronous refresh: no store read on this (gRPC request) thread at all. + if (_refreshed.TryGetValue(clusterId, out var refreshed)) + { + psk = refreshed; + return true; + } + var entry = _cache.GetOrAdd( clusterId, static (id, self) => new Lazy( @@ -288,7 +325,85 @@ public sealed class MeshPskProvider : IMeshPskProvider } /// - public void InvalidateAll() => _cache.Clear(); + public void InvalidateAll() + { + _cache.Clear(); + _refreshed = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + /// + public async Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(clusterIds); + + var previous = _refreshed; + var next = new Dictionary(StringComparer.OrdinalIgnoreCase); + var ids = clusterIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + + // Resolve every cluster concurrently — each read bounded like the synchronous path — and carry a + // failed cluster's PREVIOUS refreshed key forward: a transient store fault must not turn into a + // window in which that cluster has no key at all. + var reads = ids.Select(async id => + { + var value = await ResolveFromStoreAsync(id, ct).ConfigureAwait(false); + return (Id: id, Value: value); + }).ToList(); + foreach (var (id, value) in await Task.WhenAll(reads).ConfigureAwait(false)) + { + if (!string.IsNullOrWhiteSpace(value)) next[id] = value; + else if (previous.TryGetValue(id, out var kept)) next[id] = kept; + } + + // The swap. Then drop the lazy entries for the clusters now covered, so a key rotated between + // refreshes is served from the new snapshot rather than from a stale lazy resolution; entries + // for clusters NOT covered (their read failed and they had no previous value) are left alone. + _refreshed = next; + foreach (var id in next.Keys) _cache.TryRemove(id, out _); + } + + /// + /// The asynchronous twin of : same name derivation, same bound, same + /// "anything but a usable key is null" contract — awaited instead of blocked on, because this one + /// runs on a refresh rather than inside a gRPC interceptor. + /// + /// The application Cluster id. + /// The caller's token, linked under the store-read bound. + /// The stored key, or null. + private async Task ResolveFromStoreAsync(string clusterId, CancellationToken ct) + { + SecretName name; + try + { + name = new SecretName(SecretNamePrefix + clusterId); + } + catch (ArgumentException ex) + { + _logger.LogWarning( + ex, + "Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be " + + "refreshed from the store. Supply it through MeshTransport:ClusterPsks instead, or " + + "rename the cluster to [a-z0-9._/-].", + clusterId); + return null; + } + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + cts.CancelAfter(StoreReadTimeout); + try + { + return await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Failed to refresh the mesh PSK for cluster {ClusterId} from secret {SecretName}; the " + + "previously resolved key (if any) stays in service until the next refresh.", + clusterId, + name.Value); + return null; + } + } /// /// Reads OT-MESH-PSK-{clusterId} from the Secrets store, converting every failure mode diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs index a10d228a..31a19b13 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs @@ -34,6 +34,15 @@ public interface IMeshCommandDialer /// rather than dialled unauthenticated. /// Task DispatchAsync(string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct); + + /// + /// Releases any per-endpoint resources for endpoints NOT in + /// (R03:1.3). Central calls it on every contact refresh with the current dispatch set, so a + /// renamed host or moved port does not leave the old channel and its idle HTTP/2 connection + /// alive for the life of the process. Default no-op for dialers that cache nothing. + /// + /// The endpoints still in the dispatch set. + void RetainOnly(IReadOnlyCollection nodeEndpoints) { } } /// @@ -181,6 +190,30 @@ public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable await call.ResponseAsync.ConfigureAwait(false); } + /// + public void RetainOnly(IReadOnlyCollection nodeEndpoints) + { + ArgumentNullException.ThrowIfNull(nodeEndpoints); + if (_disposed) return; + + var keep = nodeEndpoints as IReadOnlySet ?? nodeEndpoints.ToHashSet(StringComparer.Ordinal); + foreach (var endpoint in _channels.Keys) + { + if (keep.Contains(endpoint)) continue; + // A dial that already took this channel out of the cache before the removal fails once, as + // a named per-node RpcException the actor logs; the next dial to that endpoint (if it ever + // returns to the set) builds a fresh channel. Bounded to the dials in flight at the instant + // of a contact refresh, the same accepted race Dispose documents. + if (_channels.TryRemove(endpoint, out var channel)) + { + channel.Dispose(); + _log.LogInformation( + "Mesh dialer released the cached channel for {Endpoint}: it is no longer in the dispatch set", + endpoint); + } + } + } + /// Marks the dialer disposed and disposes every cached gRPC channel. public void Dispose() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs index 9843c8fe..51dcd046 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs @@ -930,6 +930,34 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase /// A spawned gRPC-mode actor plus the two seams the tests observe it through. private sealed record GrpcRig(IActorRef Actor, RecordingDialer Dialer, RecordingPskProvider Psk); + /// + /// R03:1.3 — every contact refresh hands the dialer the CURRENT endpoint set so it can release the + /// channels of endpoints that left it. + /// + [Fact] + public void Each_contact_refresh_tells_the_dialer_which_endpoints_remain() + { + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false), ("host-b:4053", "host-b", 4056, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + // The dialer is resolved lazily on first use; a dispatch forces that, and the NEXT refresh prunes. + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5)); + + using (var db = factory.CreateDbContext()) + { + db.ClusterNodes.Single(n => n.NodeId == "host-b:4053").MaintenanceMode = true; + db.SaveChanges(); + } + rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts()); + + AwaitCondition( + () => rig.Dialer.RetainCalls.Any(set => set.Count == 1 && set.Single().Contains("host-a")), + TimeSpan.FromSeconds(5)); + rig.Dialer.RetainCalls.Last().ShouldBe(new[] { "http://host-a:4056" }); + } + private GrpcRig SpawnGrpcRig( IDbContextFactory factory, RecordingDialer? dialer = null, @@ -1014,6 +1042,19 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase : Task.CompletedTask; } + /// Every endpoint set handed to , in order. + public IReadOnlyList> RetainCalls + { + get { lock (_gate) return _retainCalls.ToList(); } + } + + private readonly List> _retainCalls = []; + + public void RetainOnly(IReadOnlyCollection nodeEndpoints) + { + lock (_gate) _retainCalls.Add(nodeEndpoints.ToList()); + } + /// One recorded dial. public sealed record Dial(string Endpoint, CommandEnvelope Envelope, string ClusterId); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs index 2d110eb6..fc177361 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs @@ -224,6 +224,35 @@ public sealed class MeshGrpcCommandDialerTests Should.NotThrow(dialer.Dispose); } + /// + /// R03:1.3 — a channel for an endpoint that left the dispatch set (host renamed, port moved) is + /// released on the next refresh instead of living, with its idle connection, for the life of central. + /// Positive control: a dial to that endpoint afterwards rebuilds a channel. + /// + [Fact] + public async Task RetainOnly_releases_channels_for_endpoints_that_left_the_dispatch_set() + { + var dialer = new MeshGrpcCommandDialer( + new FakePskProvider(("SITE-A", "psk-site-a")), + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }), + NullLogger.Instance); + foreach (var endpoint in new[] { "http://127.0.0.1:1", "http://127.0.0.1:2" }) + { + await Should.ThrowAsync( + () => dialer.DispatchAsync(endpoint, Envelope(), "SITE-A", CancellationToken.None)); + } + dialer.CachedChannelCount.ShouldBe(2); + + dialer.RetainOnly(new[] { "http://127.0.0.1:1" }); + + dialer.CachedChannelCount.ShouldBe(1); + + await Should.ThrowAsync( + () => dialer.DispatchAsync("http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None)); + dialer.CachedChannelCount.ShouldBe(2, "a returning endpoint gets a fresh channel"); + Should.NotThrow(dialer.Dispose); + } + [Fact] public void It_activates_from_the_container_despite_the_optional_client_factory_parameter() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs new file mode 100644 index 00000000..e727c153 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs @@ -0,0 +1,129 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; +using ZB.MOM.WW.Secrets.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; + +/// +/// R03:3.4 — central invalidated every cached key on every contact refresh (default 60 s), so the +/// first inbound ack per cluster per minute blocked a gRPC thread on a synchronous, 3 s-bounded store +/// read and was denied past the bound. pre-resolves the +/// keys off the request path and swaps them in atomically. +/// +public sealed class MeshPskProviderRefreshTests +{ + private static MeshPskProvider Provider(ISecretResolver resolver) => + new(Options.Create(new MeshTransportOptions()), resolver, NullLogger.Instance); + + [Fact] + public async Task After_a_refresh_the_lookup_is_served_without_touching_the_store() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + resolver.Calls.ShouldBe(1); + + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-1"); + resolver.Calls.ShouldBe(1, "the request path did not consult the store"); + provider.Matches("SITE-A", "k-1").ShouldBeTrue(); + } + + /// + /// While a refresh is in flight the PREVIOUS key is still served (no invalidate-then-reread window); + /// once it completes the rotated key is served. Positive control on both sides of the swap. + /// + [Fact] + public async Task A_rotated_key_is_swapped_in_atomically_with_the_old_key_served_until_then() + { + var resolver = new GatedResolver { Value = "old" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + provider.TryGetPsk("SITE-A", out var before).ShouldBeTrue(); + before.ShouldBe("old"); + + resolver.Value = "new"; + resolver.Hold(); + var refresh = provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + resolver.Entered.Wait(TimeSpan.FromSeconds(5)).ShouldBeTrue(); + + // Mid-refresh: the old key, from memory — the store is NOT re-read on this path. + var callsMidRefresh = resolver.Calls; + provider.TryGetPsk("SITE-A", out var during).ShouldBeTrue(); + during.ShouldBe("old"); + resolver.Calls.ShouldBe(callsMidRefresh); + + resolver.Release(); + await refresh; + provider.TryGetPsk("SITE-A", out var after).ShouldBeTrue(); + after.ShouldBe("new"); + } + + [Fact] + public async Task A_cluster_whose_refresh_fails_keeps_its_previous_key() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + resolver.Throw = new InvalidOperationException("store offline"); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-1"); + } + + /// A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour). + [Fact] + public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily() + { + var resolver = new GatedResolver { Value = "k-b" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-B", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-b"); + resolver.Calls.ShouldBe(2); + } + + [Fact] + public async Task InvalidateAll_drops_refreshed_keys_too() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.InvalidateAll(); + resolver.Value = "k-2"; + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-2"); + } + + private sealed class GatedResolver : ISecretResolver + { + private int _calls; + private volatile TaskCompletionSource? _gate; + + public string? Value { get; set; } + public Exception? Throw { get; set; } + public int Calls => Volatile.Read(ref _calls); + public ManualResetEventSlim Entered { get; } = new(); + + public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _gate?.TrySetResult(); + + public async Task GetAsync(SecretName name, CancellationToken ct) + { + Interlocked.Increment(ref _calls); + Entered.Set(); + if (_gate is { } gate) await gate.Task.WaitAsync(ct); + if (Throw is not null) throw Throw; + return Value; + } + } +} From e04156d8f7006067518955358281566f199939e1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:39:12 -0400 Subject: [PATCH 09/19] fix(host): refuse unknown API-key ids in memory and budget store misses; document the mesh gate's boot window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BoundedApiKeyStore's negative cache was keyed by key id, so a flood of DISTINCT well-formed otua__x credentials missed it every time and cost one SQL query per attempt under the SDK's SessionManager-wide lock, no credential needed (R03:3.3). - IApiKeyStore.ListKeyIdsAsync (default null = unsupported; ConfigDbApiKeyStore lists every ApiKeyId): the bounded store keeps a known-id set refreshed every StoreCacheSeconds (single-flight, failed refresh keeps the previous set) and refuses an unknown id without a store read ({outcome=unknown_id}). - A process-wide miss budget (token bucket, Security:ApiKeys:MissBurst=100 / MissRefillPerSecond=20, ApiKeyStoreGuardOptions) bounds lookups that reach the store and miss while no set is available; a hit refunds its token. Per-endpoint accounting needs the OpcUaServer impersonation seam to carry the client address — follow-up. - docs/security.md: the new knobs, and the known-cluster gate's documented fail-open window at central boot (R03:4.2) plus the async PSK refresh (R03:3.4). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/security.md | 44 +++++ .../DataPlaneAuthRegistration.cs | 5 +- .../OpcUa/ApiKeyStoreGuardOptions.cs | 56 ++++++ .../OpcUa/BoundedApiKeyStore.cs | 117 +++++++++++- .../OpcUa/ConfigDbApiKeyStore.cs | 10 + .../OpcUa/IApiKeyStore.cs | 10 + .../OpcUa/BoundedApiKeyStoreKnownIdTests.cs | 173 ++++++++++++++++++ .../OpcUa/BoundedApiKeyStoreTests.cs | 11 +- 8 files changed, 421 insertions(+), 5 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs diff --git a/docs/security.md b/docs/security.md index 87ac8d22..6e44569a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -245,6 +245,25 @@ with two `Security:ApiKeys` knobs: a completed lookup (a hit or an authoritative miss) is stored, and a cached hit whose `ExpiresUtc` has since passed is re-read rather than served, so the TTL can never resurrect an expired key. +#### Pre-authentication guards (`KnownIdSet` / `MissBurst` / `MissRefillPerSecond`) + +The negative cache above is keyed by **key id**, so a flood of *distinct*, well-formed `otua__x` +credentials missed it every time and cost one indexed SQL query per attempt — under the SDK's +`SessionManager`-wide lock, with no credential required (review 03 §3.3). `BoundedApiKeyStore` now puts two +guards in front of the store, bound from the same `Security:ApiKeys` section +(`ApiKeyStoreGuardOptions`, Host-owned): + +| Key | Default | Meaning | +|---|---|---| +| `KnownIdSet` | `true` | Keep an in-memory set of every `ApiKeyId` the ConfigDb holds, refreshed every `StoreCacheSeconds` (single-flight; a failed refresh keeps the previous set). An id outside it is refused in memory — `otopcua.opcua.apikey_store.bounded{outcome=unknown_id}` — and the store is never asked. A key minted less than `StoreCacheSeconds` ago is refused until the next refresh (the same window `Enabled` revocation already has). `StoreCacheSeconds = 0` disables the set with the cache. | +| `MissBurst` | `100` | Token bucket on lookups that **reach** the store and **miss**, for when the set is unavailable (not loaded yet, or the store cannot list ids). Exhausted ⇒ unknown ids are refused without a store read (`{outcome=miss_budget}`, a Warning-free deny through the authenticator's catch-all). A hit refunds its token, so legitimate clients are never charged. `0` disables the budget. | +| `MissRefillPerSecond` | `20` | Refill rate of the miss bucket. | + +The budget is **process-wide, not per remote endpoint**: the OPC UA SDK's impersonation callback (the +`IOpcUaUserAuthenticator` seam) does not carry the client's address, so per-endpoint accounting needs that +seam widened in the OpcUaServer project (a Wave 3 follow-up). Neither guard applies to a driver-only node: +its `ArtifactApiKeyStore` is two in-memory dictionary probes and is never wrapped. + ### LDAP bind flow (`OtOpcUaLdapAuthService`) LDAP is configured under the `Security:Ldap` section (bound to `LdapOptions`, `src/Server/ZB.MOM.WW.OtOpcUa.Security/Ldap/LdapOptions.cs`, `SectionName = "Security:Ldap"`). The app authenticator is `OtOpcUaLdapAuthService` — a thin wrapper around the shared `ZB.MOM.WW.Auth.Ldap` directory client that adds two app-only concerns the shared library deliberately does not model: the `Enabled` master switch and `DevStubMode`. Since #520 it serves the **Admin UI cookie login (`/auth/login`) only** — `LdapOpcUaUserAuthenticator` is deleted and the OPC UA UserName path authenticates API keys instead (see [API-key identity](#api-key-identity-opc-ua-data-plane)). @@ -1116,6 +1135,31 @@ See `archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md` (R3-01) for the fu record, and `docs/Configuration.md` §`MeshTransport` / §`ConfigSource`-`ConfigServe` and `docs/Telemetry.md` for the affected listener ports. +### The known-cluster gate is fail-OPEN until central's first ClusterNode load (review 03 §4.2) + +`CentralCommandAuthInterceptor` verifies an inbound deploy ack against the PSK of the cluster named in +`x-otopcua-cluster`. `MeshPskProvider` consults a **known-cluster gate** first — the set of cluster ids +central read from `ClusterNode` on its last contact refresh — and refuses anything outside it in O(1), +touching neither the config map nor the secret store. That gate is **inactive until the first successful +load** (`CentralCommunicationActor` publishes it from `ContactsLoaded`; an empty publication also +deactivates it, deliberately, so "the DB told me nothing" never becomes a fleet-wide authentication +outage). During a central boot with the ConfigDb down, therefore: + +- an unauthenticated caller can name *any* `x-otopcua-cluster` value and make central perform one bounded + (3 s) secret-store read plus one Warning line per request — the cost/oracle the gate exists to remove; +- **keys are still required**: nothing is admitted without the matching PSK, so this is a cost exposure, + not an authentication bypass; +- the window is bounded by the bootstrap retry (15 × 2 s) and the `ContactRefreshSeconds` (60 s) periodic + refresh, and closes on the first load that succeeds. The actor logs the condition on every failed load + (*"The known-cluster gate was NOT refreshed either … it is INACTIVE"*). + +Since review 03 §3.4 the per-cluster keys are **refreshed asynchronously** from the same handler +(`IMeshPskProvider.RefreshAsync`: concurrent bounded reads, atomic snapshot swap, a failed read keeps the +previous key) instead of being invalidated on every refresh, so the first ack per cluster per minute no +longer pays a synchronous store read on a gRPC thread. Accepted as documented, bounded behaviour; the +alternative — refusing every cluster until SQL answers — would turn a ConfigDb outage into a deploy-ack +outage on top of it. + --- ## OTOPCUA0001 Analyzer — Compile-Time Guard diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs index e7c301e4..eace8505 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs @@ -80,7 +80,10 @@ public static class DataPlaneAuthRegistration sp.GetRequiredService>().Value)), sp.GetRequiredService>(), sp.GetRequiredService().CreateLogger(), - sp.GetRequiredService())); + sp.GetRequiredService(), + // R03:3.3 — the known-id set + miss budget in front of the SQL lookup; same section as + // ApiKeyOptions, Host-owned knobs. + ApiKeyStoreGuardOptions.Bind(sp.GetService()))); services.AddSingleton(); // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs new file mode 100644 index 00000000..3103b982 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs @@ -0,0 +1,56 @@ +using Microsoft.Extensions.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// +/// The pre-authentication guards puts in front of the ConfigDb key +/// lookup (R03:3.3). Bound from the same Security:ApiKeys section as ApiKeyOptions but +/// owned by the Host, where the store lives. +/// +/// +/// +/// Why. The negative cache was keyed by key id, so a flood of DISTINCT well-formed +/// otua_<random>_x credentials missed it every time and cost one indexed SQL query per +/// attempt — under the SDK's SessionManager-wide lock — with no credential needed. Two guards close +/// that: a known-id set (every ApiKeyId the ConfigDb holds, refreshed every +/// StoreCacheSeconds) rejects an unknown id in memory, and a miss budget (token +/// bucket) bounds how many lookups that reach the store and MISS are allowed per second when the +/// set is unavailable. +/// +/// +/// The budget is process-wide, not per remote endpoint: the OPC UA SDK's impersonation callback +/// (the IOpcUaUserAuthenticator seam) does not carry the client's address. Per-endpoint +/// accounting needs that seam widened in the OpcUaServer project — a follow-up, not this lane. +/// +/// +public sealed class ApiKeyStoreGuardOptions +{ + /// The configuration section the knobs live in (shared with ApiKeyOptions). + public const string SectionName = "Security:ApiKeys"; + + /// + /// Whether the known-id set is consulted. restores a store read per unknown + /// id. Only meaningful when the inner store can list ids (the ConfigDb store can; the artifact + /// store is never wrapped). + /// + public bool KnownIdSet { get; set; } = true; + + /// + /// How many store MISSES (lookups that reach the inner store and find no key) may burst before + /// further lookups for unknown ids are refused without a store read. 0 disables the budget. + /// + public int MissBurst { get; set; } = 100; + + /// How many miss tokens are restored per second, up to . + public double MissRefillPerSecond { get; set; } = 20; + + /// Binds the section, with defaults for anything absent. + /// The app configuration; null yields the defaults. + /// The bound options. + public static ApiKeyStoreGuardOptions Bind(IConfiguration? configuration) + { + var options = new ApiKeyStoreGuardOptions(); + configuration?.GetSection(SectionName).Bind(options); + return options; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index af0fca3f..bb3154ed 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -56,6 +56,21 @@ public sealed class BoundedApiKeyStore : IApiKeyStore private readonly ConcurrentDictionary> _keys = new(StringComparer.Ordinal); private readonly ConcurrentDictionary>> _grants = new(StringComparer.Ordinal); + private readonly ApiKeyStoreGuardOptions _guard; + + // R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds + // (single-flight; a refresh that fails or times out keeps the previous set). Null until the first + // successful load, or when the inner store cannot list ids — then the per-id path runs as before. + private volatile IReadOnlySet? _knownIds; + private DateTimeOffset _knownIdsStampedUtc = DateTimeOffset.MinValue; + private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1); + + // R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards + // the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the + // configured rate instead of one SQL round trip per attempt under the SessionManager lock. + private readonly object _bucketGate = new(); + private double _missTokens; + private DateTimeOffset _missRefilledUtc; /// Initializes a new instance of the class. /// The store being bounded. @@ -66,11 +81,13 @@ public sealed class BoundedApiKeyStore : IApiKeyStore /// DataPlaneAuthRegistration for why a second clock here would be a hazard rather than a /// detail. /// + /// The pre-authentication guards (R03:3.3); defaults when omitted. public BoundedApiKeyStore( IApiKeyStore inner, IOptions options, ILogger log, - TimeProvider time) + TimeProvider time, + ApiKeyStoreGuardOptions? guard = null) { ArgumentNullException.ThrowIfNull(inner); ArgumentNullException.ThrowIfNull(options); @@ -81,8 +98,14 @@ public sealed class BoundedApiKeyStore : IApiKeyStore _options = options; _log = log; _time = time; + _guard = guard ?? new ApiKeyStoreGuardOptions(); + _missTokens = Math.Max(0, _guard.MissBurst); + _missRefilledUtc = time.GetUtcNow(); } + /// The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards. + public IReadOnlySet? KnownKeyIds => _knownIds; + /// The store being bounded. /// /// Exposed so the composition root's wiring can be asserted. A decorator registered in one branch @@ -115,12 +138,104 @@ public sealed class BoundedApiKeyStore : IApiKeyStore return cached; } + // R03:3.3, guard 1 — an id the ConfigDb does not hold is refused in memory; the store is never asked. + var known = await KnownIdsAsync(ttl, ct).ConfigureAwait(false); + if (known is not null && !known.Contains(apiKeyId)) + { + OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair("outcome", "unknown_id")); + return null; + } + + // R03:3.3, guard 2 — a lookup that will reach the store needs a miss token unless the id is known + // to exist (a known id is a real key being verified, never a guess). + if (known is null && !TryTakeMissToken()) + { + OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair("outcome", "miss_budget")); + throw new InvalidOperationException( + "ApiKeyStore: the unknown-key miss budget is exhausted (Security:ApiKeys:MissBurst / " + + "MissRefillPerSecond); refusing this lookup without a store read."); + } + var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct) .ConfigureAwait(false); + if (fresh is not null && known is null) RefundMissToken(); if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh); return fresh; } + /// + /// The known-id set, refreshed when older than . Single-flight: concurrent + /// callers during a refresh use the previous set (or, before the first load, the per-id path). A + /// refresh that fails or exceeds the deadline logs and keeps the previous set. + /// + /// The refresh interval (the cache TTL); non-positive disables the set. + /// The caller's token. + /// The set, or null. + private async Task?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct) + { + if (!_guard.KnownIdSet || ttl <= TimeSpan.Zero) return null; + + var now = _time.GetUtcNow(); + var current = _knownIds; + if (current is not null && now - _knownIdsStampedUtc < ttl) return current; + + if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current; + try + { + // Re-check under the gate: another caller may have refreshed while this one waited. + if (_knownIds is { } refreshed && now - _knownIdsStampedUtc < ttl) return refreshed; + var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false); + if (ids is null) + { + // Unsupported by the inner store: say so once and stop asking. + _guard.KnownIdSet = false; + return null; + } + _knownIds = ids; + _knownIdsStampedUtc = _time.GetUtcNow(); + return ids; + } + catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) + { + _log.LogWarning(ex, + "ApiKeyStore: refreshing the known key-id set failed; {Mode}", + current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force"); + return current; + } + finally + { + _knownIdsRefresh.Release(); + } + } + + /// Takes one miss token, refilling by elapsed time first; always succeeds when the budget is disabled. + /// when the budget is exhausted. + private bool TryTakeMissToken() + { + var burst = _guard.MissBurst; + if (burst <= 0) return true; + lock (_bucketGate) + { + var now = _time.GetUtcNow(); + var elapsed = (now - _missRefilledUtc).TotalSeconds; + if (elapsed > 0) + { + _missTokens = Math.Min(burst, _missTokens + elapsed * Math.Max(0, _guard.MissRefillPerSecond)); + _missRefilledUtc = now; + } + if (_missTokens < 1) return false; + _missTokens -= 1; + return true; + } + } + + /// A lookup that reached the store and HIT was not a miss: its token goes back. + private void RefundMissToken() + { + if (_guard.MissBurst <= 0) return; + lock (_bucketGate) _missTokens = Math.Min(_guard.MissBurst, _missTokens + 1); + } + // Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect. /// public async Task> FindGrantsAsync(string principal, CancellationToken ct) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs index c3f9c1f5..52fcbe51 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs @@ -125,6 +125,16 @@ public sealed class ConfigDbApiKeyStore : IApiKeyStore return await LoadGrantsAsync(db, principal, ct).ConfigureAwait(false); } + /// + public async Task?> ListKeyIdsAsync(CancellationToken ct) + { + await using var db = await _factory.CreateDbContextAsync(ct).ConfigureAwait(false); + // One projection over the whole (small) table — the unique index on ApiKeyId makes this an index + // scan — so an unknown id can be refused in memory instead of costing a query per attempt. + var ids = await db.ApiKeys.AsNoTracking().Select(k => k.ApiKeyId).ToListAsync(ct).ConfigureAwait(false); + return ids.ToHashSet(StringComparer.Ordinal); + } + /// Loads a principal's grants, confined to this node's cluster. /// /// The full grant set, not just the Cluster-scoped rows. The old diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs index c2f82dae..93057153 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs @@ -35,6 +35,16 @@ public interface IApiKeyStore /// The cancellation token. /// The matching ACL rows; empty when none. Task> FindGrantsAsync(string principal, CancellationToken ct); + + /// + /// Every ApiKeyId the store holds (enabled or not), for the pre-authentication known-id set + /// (R03:3.3); when the store cannot cheaply list them, in which case callers + /// fall back to a per-id lookup. The set only answers "does this id exist at all" — enabled, expiry + /// and the secret are still decided by . + /// + /// The cancellation token. + /// The id set, or null when unsupported. + Task?> ListKeyIdsAsync(CancellationToken ct) => Task.FromResult?>(null); } /// An API key resolved for authentication, with the ACL grants naming its principal. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs new file mode 100644 index 00000000..7051b151 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs @@ -0,0 +1,173 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// R03:3.3 — the negative cache was keyed by key id, so a flood of DISTINCT well-formed ids cost one +/// SQL query per attempt under the SessionManager lock, no credential needed. Two guards: +/// the known-id set (unknown ids refused in memory) and the miss budget (a token bucket on lookups +/// that reach the store and miss, for when the set is unavailable). +/// +public sealed class BoundedApiKeyStoreKnownIdTests +{ + private static readonly DateTimeOffset T0 = new(2026, 10, 22, 12, 0, 0, TimeSpan.Zero); + + [Fact] + public async Task An_unknown_id_is_refused_without_reaching_the_store_and_a_known_one_still_does() + { + var inner = new ListingStore("k-real"); + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + (await sut.FindAsync("k-guess-1", T0.UtcDateTime, default)).ShouldBeNull(); + (await sut.FindAsync("k-guess-2", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(0, "unknown ids never reach the store"); + inner.ListCalls.ShouldBe(1, "one set load serves every unknown id inside the TTL"); + + // Positive control: the real id reaches the store and is found. + (await sut.FindAsync("k-real", T0.UtcDateTime, default)).ShouldNotBeNull(); + inner.FindCalls.ShouldBe(1); + } + + [Fact] + public async Task The_set_is_refreshed_after_the_TTL_so_a_newly_minted_key_becomes_visible() + { + var inner = new ListingStore("k-old"); + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + (await sut.FindAsync("k-new", T0.UtcDateTime, default)).ShouldBeNull(); + inner.Mint("k-new"); + // Still inside the TTL: the stale set refuses it (the documented window). + clock.Advance(TimeSpan.FromSeconds(3)); + (await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(0); + + clock.Advance(TimeSpan.FromSeconds(3)); + (await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + inner.ListCalls.ShouldBe(2); + } + + [Fact] + public async Task A_failed_set_load_falls_back_to_the_per_id_lookup_and_keeps_an_older_set() + { + var inner = new ListingStore("k-real") { ListThrows = true }; + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + // No set yet and the load fails: the per-id path runs (old behaviour), nothing is wedged. + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(1); + sut.KnownKeyIds.ShouldBeNull(); + + inner.ListThrows = false; + clock.Advance(TimeSpan.FromSeconds(6)); + (await sut.FindAsync("k-guess-2", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + sut.KnownKeyIds.ShouldNotBeNull(); + inner.FindCalls.ShouldBe(1, "once the set loaded, the guess was refused in memory"); + + // A later failed refresh keeps the previous set in force. + inner.ListThrows = true; + clock.Advance(TimeSpan.FromSeconds(6)); + (await sut.FindAsync("k-guess-3", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(1); + } + + /// With no id set available, misses are rate-limited; hits are not charged; time refills. + [Fact] + public async Task The_miss_budget_bounds_store_misses_when_no_set_is_available() + { + var inner = new ListingStore("k-real") { SupportsListing = false }; + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 0, missBurst: 3, refillPerSecond: 1); + + for (var i = 0; i < 3; i++) + (await sut.FindAsync($"k-guess-{i}", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(3); + + // Budget exhausted: refused without a store read, as a throw the authenticator denies on. + await Should.ThrowAsync(() => sut.FindAsync("k-guess-9", T0.UtcDateTime, default)); + inner.FindCalls.ShouldBe(3); + + // Refill: one second restores one token. + clock.Advance(TimeSpan.FromSeconds(1)); + (await sut.FindAsync("k-guess-10", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(4); + + // A hit is not a miss: it reaches the store even with the bucket empty? No — it needs a token to + // be attempted, but the token is refunded, so the next attempt still has it. Prove the refund. + clock.Advance(TimeSpan.FromSeconds(1)); + (await sut.FindAsync("k-real", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + (await sut.FindAsync("k-guess-11", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(6); + } + + [Fact] + public async Task The_guards_are_off_when_the_cache_is_off_restoring_a_read_per_lookup() + { + var inner = new ListingStore("k-real"); + var sut = Bounded(inner, new FakeClock(T0), cacheSeconds: 0, missBurst: 0); + + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(2); + inner.ListCalls.ShouldBe(0); + } + + private static BoundedApiKeyStore Bounded( + IApiKeyStore inner, TimeProvider clock, int cacheSeconds, int missBurst = 100, double refillPerSecond = 20) => new( + inner, + Options.Create(new ApiKeyOptions + { + Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = cacheSeconds, + }), + NullLogger.Instance, + clock, + new ApiKeyStoreGuardOptions { MissBurst = missBurst, MissRefillPerSecond = refillPerSecond }); + + private sealed class ListingStore(params string[] ids) : IApiKeyStore + { + private readonly HashSet _ids = ids.ToHashSet(StringComparer.Ordinal); + private int _findCalls; + private int _listCalls; + + public bool SupportsListing { get; set; } = true; + public bool ListThrows { get; set; } + public int FindCalls => _findCalls; + public int ListCalls => _listCalls; + + public void Mint(string id) => _ids.Add(id); + + public Task FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct) + { + Interlocked.Increment(ref _findCalls); + return Task.FromResult(_ids.Contains(apiKeyId) + ? new ApiKeyLookup(new ApiKey { ApiKeyId = apiKeyId, Name = "k", SecretHash = "h", Principal = "p" }, []) + : null); + } + + public Task> FindGrantsAsync(string principal, CancellationToken ct) => + Task.FromResult>([]); + + public Task?> ListKeyIdsAsync(CancellationToken ct) + { + if (!SupportsListing) return Task.FromResult?>(null); + Interlocked.Increment(ref _listCalls); + if (ListThrows) throw new InvalidOperationException("ConfigDb unreachable (simulated)"); + return Task.FromResult?>(_ids.ToHashSet(StringComparer.Ordinal)); + } + } + + private sealed class FakeClock(DateTimeOffset utcNow) : TimeProvider + { + private DateTimeOffset _now = utcNow; + public override DateTimeOffset GetUtcNow() => _now; + public void Advance(TimeSpan by) => _now = _now.Add(by); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs index e8081db1..65201905 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs @@ -363,7 +363,10 @@ public sealed class BoundedApiKeyStoreTests { var (_, inner) = SetUp(); var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); - var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock); + // The miss budget (R03:3.3) is OFF here on purpose: this fact floods 1025 distinct unknown ids — the + // exact shape the budget refuses — to pin the cache CEILING, which is a separate rule. + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock, + guard: new ApiKeyStoreGuardOptions { MissBurst = 0 }); // Seed the real key, then flood past the ceiling with ids that do not exist. (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); @@ -477,7 +480,8 @@ public sealed class BoundedApiKeyStoreTests int timeoutMs, int cacheSeconds, TimeProvider? clock = null, - ILogger? log = null) => new( + ILogger? log = null, + ApiKeyStoreGuardOptions? guard = null) => new( inner, Options.Create(new ApiKeyOptions { @@ -487,7 +491,8 @@ public sealed class BoundedApiKeyStoreTests StoreCacheSeconds = cacheSeconds, }), log ?? NullLogger.Instance, - clock ?? TimeProvider.System); + clock ?? TimeProvider.System, + guard); private static ApiKeyOpcUaUserAuthenticator Auth(IApiKeyStore store) => new( store, From 3ddb6876dbe65660c7b2f5e7a5fb3eba4a870151 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:35:54 -0400 Subject: [PATCH 10/19] fix(security): first-party JwtBearer with HS256 pinned; /auth/token mints for a cookie session only The hand-rolled bearer handler is replaced by AddJwtBearer (Microsoft.AspNetCore.Authentication.JwtBearer 10.0.7, pinned on the same servicing line as the other AspNetCore packages). Validation parameters come from JwtTokenService.BuildValidationParameters, which now pins ValidAlgorithms = HS256 for both the bearer scheme and TryValidate; MapInboundClaims is off so the minted Role/zb:username keys resolve. /auth/token requires the cookie scheme, so a bearer can no longer mint a fresh token for itself. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- Directory.Packages.props | 4 + docs/security.md | 13 ++- .../Auth/JwtBearerAuthenticationHandler.cs | 99 ------------------- .../Auth/OtOpcUaAuthSchemes.cs | 39 ++++++++ .../Endpoints/AuthEndpoints.cs | 16 ++- .../Jwt/JwtTokenService.cs | 28 ++---- .../ServiceCollectionExtensions.cs | 18 +++- .../ZB.MOM.WW.OtOpcUa.Security.csproj | 1 + .../AuthEndpointsIntegrationTests.cs | 93 +++++++++++++++++ .../JwtConfigurationGateTests.cs | 2 +- 10 files changed, 187 insertions(+), 126 deletions(-) delete mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 5b95a7e1..ed2dca67 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -35,6 +35,10 @@ + + diff --git a/docs/security.md b/docs/security.md index 6e44569a..fa8269a2 100644 --- a/docs/security.md +++ b/docs/security.md @@ -6,11 +6,14 @@ > auth + role-grant pages live in `OtOpcUa.AdminUI`), and Admin auth policies > register from `OtOpcUa.Host/Program.cs` via `AddOtOpcUaAuth` > (`src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs`) rather -> than in a separate Admin process. The Admin UI uses a **single Cookie -> authentication scheme** — there is no `AddJwtBearer` pipeline. The -> `Security:Jwt` section configures `JwtTokenService`, which mints a JWT at the -> `/auth/token` endpoint for **external** consumers (OPC UA clients / automation -> scripts); the cookie itself stores the `ClaimsPrincipal` directly. DataProtection +> than in a separate Admin process. The Admin UI session is a **Cookie** +> scheme; the cookie stores the `ClaimsPrincipal` directly. JWT is **opt-in by +> `Security:Jwt:SigningKey`** (validated at host start, ≥32 bytes): with a key, +> `/auth/token` mints an HS256 token for a **cookie session only** (a bearer +> cannot renew itself), and a first-party `AddJwtBearer` scheme (`Bearer`, +> `ValidAlgorithms = HS256`, parameters from `JwtTokenService.BuildValidationParameters`) +> accepts it on `/api/*` and every `RequireAuthorization` endpoint through the +> `OtOpcUa.CookieOrBearer` default policy scheme. Without a key neither exists. DataProtection > keys persist to the shared Config DB (`PersistKeysToDbContext`) > so cookies survive failover between admin-role nodes. > diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs deleted file mode 100644 index cc996758..00000000 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs +++ /dev/null @@ -1,99 +0,0 @@ -using System.Text.Encodings.Web; -using Microsoft.AspNetCore.Authentication; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Primitives; -using ZB.MOM.WW.OtOpcUa.Security.Jwt; - -namespace ZB.MOM.WW.OtOpcUa.Security.Auth; - -/// The authentication scheme names OtOpcUa registers. -public static class OtOpcUaAuthSchemes -{ - /// - /// The bearer scheme that validates the HS256 tokens /auth/token mints, through - /// . Registered only when - /// . - /// - public const string Bearer = "Bearer"; - - /// - /// The default scheme: a policy scheme that forwards to when the request - /// carries an Authorization: Bearer header and to the cookie scheme otherwise, so one - /// RequireAuthorization() / policy accepts a browser session AND a machine token. - /// - public const string CookieOrBearer = "OtOpcUa.CookieOrBearer"; - - /// Whether the request presents a bearer token (the selector's whole rule). - /// The request. - /// when an Authorization: Bearer … header is present. - public static bool HasBearerToken(HttpContext context) => - TryGetBearerToken(context.Request.Headers.Authorization, out _); - - internal static bool TryGetBearerToken(StringValues authorization, out string token) - { - token = string.Empty; - var header = authorization.ToString(); - const string prefix = "Bearer "; - if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; - token = header[prefix.Length..].Trim(); - return token.Length > 0; - } -} - -/// -/// Validates an Authorization: Bearer token minted by /auth/token (R03:2.4). The JWT -/// feature used to be a dead end: tokens were issued and nothing in the product accepted them. This -/// handler is the consuming side, built on so the validation -/// parameters can never drift from the minting side ( -/// sets RoleClaimType/NameClaimType to the minted keys, so [Authorize(Roles)] and -/// IsInRole resolve). -/// -/// -/// Fail closed: no header ⇒ NoResult (the cookie scheme is not consulted by this handler — the -/// policy scheme selects one or the other); a malformed, expired, tampered or wrong-issuer token ⇒ -/// Fail, never a fall-through to the cookie. A direct AddJwtBearer needs the -/// Microsoft.AspNetCore.Authentication.JwtBearer package, which the central package list does -/// not carry; this handler is ~40 lines over the already-referenced System.IdentityModel.Tokens.Jwt -/// and is a drop-in to replace once that package is pinned. -/// -public sealed class JwtBearerAuthenticationHandler : AuthenticationHandler -{ - private readonly JwtTokenService _jwt; - - /// Initializes a new instance of the class. - /// The scheme options monitor. - /// The logger factory. - /// The URL encoder. - /// The token service whose validation parameters mirror its minting side. - public JwtBearerAuthenticationHandler( - IOptionsMonitor options, - ILoggerFactory logger, - UrlEncoder encoder, - JwtTokenService jwt) - : base(options, logger, encoder) - { - _jwt = jwt; - } - - /// - protected override Task HandleAuthenticateAsync() - { - if (!OtOpcUaAuthSchemes.TryGetBearerToken(Request.Headers.Authorization, out var token)) - return Task.FromResult(AuthenticateResult.NoResult()); - - if (!_jwt.TryValidate(token, out var principal) || principal is null) - return Task.FromResult(AuthenticateResult.Fail("Invalid bearer token.")); - - return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(principal, Scheme.Name))); - } - - /// - protected override Task HandleChallengeAsync(AuthenticationProperties properties) - { - Response.StatusCode = StatusCodes.Status401Unauthorized; - Response.Headers.WWWAuthenticate = OtOpcUaAuthSchemes.Bearer; - return Task.CompletedTask; - } -} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs new file mode 100644 index 00000000..7c30915e --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs @@ -0,0 +1,39 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; + +namespace ZB.MOM.WW.OtOpcUa.Security.Auth; + +/// The authentication scheme names OtOpcUa registers. +public static class OtOpcUaAuthSchemes +{ + /// + /// The bearer scheme that validates the HS256 tokens /auth/token mints: the first-party + /// AddJwtBearer handler over . + /// Registered only when . + /// + public const string Bearer = "Bearer"; + + /// + /// The default scheme: a policy scheme that forwards to when the request + /// carries an Authorization: Bearer header and to the cookie scheme otherwise, so one + /// RequireAuthorization() / policy accepts a browser session AND a machine token. + /// + public const string CookieOrBearer = "OtOpcUa.CookieOrBearer"; + + /// Whether the request presents a bearer token (the selector's whole rule). + /// The request. + /// when an Authorization: Bearer … header is present. + public static bool HasBearerToken(HttpContext context) => + TryGetBearerToken(context.Request.Headers.Authorization, out _); + + internal static bool TryGetBearerToken(StringValues authorization, out string token) + { + token = string.Empty; + var header = authorization.ToString(); + const string prefix = "Bearer "; + if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; + token = header[prefix.Length..].Trim(); + return token.Length > 0; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs index 6b0876d7..a3f661b6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs @@ -24,6 +24,17 @@ public static class AuthEndpoints /// Response for a token issue request. public sealed record TokenResponse(string Token); + /// + /// The /auth/token policy: an authenticated principal from the COOKIE scheme. Naming the scheme + /// makes the authorization middleware authenticate that scheme alone, so an Authorization: Bearer + /// header is never consulted here (with login disabled the auto-login handler sits under the same + /// scheme name, so the dev rig is unchanged). + /// + internal static readonly AuthorizationPolicy CookieSessionOnly = + new AuthorizationPolicyBuilder(CookieAuthenticationDefaults.AuthenticationScheme) + .RequireAuthenticatedUser() + .Build(); + /// Maps OtOpcUa authentication endpoints to the application route builder. /// The endpoint route builder. /// The endpoint route builder for chaining. @@ -40,7 +51,10 @@ public static class AuthEndpoints // Mapped only when a signing key is configured (JwtOptions.IsConfigured): an unconfigured // endpoint used to answer HTTP 500 from the JwtTokenService constructor on every call. if (app.ServiceProvider.GetRequiredService>().Value.IsConfigured) - app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(); + // COOKIE scheme only (fix round 1 / I2): the default scheme also accepts a bearer, and a bearer + // that could mint a fresh token would renew itself forever — outliving its own expiry and the + // user's LDAP membership. A token is minted only for an interactive (cookie) session. + app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(CookieSessionOnly); else app.ServiceProvider.GetService()? .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs index 4dd38a2e..9dae8c14 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs @@ -25,24 +25,12 @@ public sealed class JwtTokenService /// /// Role claim type used in the JWT payload. /// - /// Issued-only / no internal JwtBearer scheme: OtOpcUa uses a single Cookie - /// authentication scheme; the JWT is minted by the /auth/token endpoint and - /// consumed externally (e.g. by OPC-UA clients or automation scripts). There is no - /// AddJwtBearer pipeline in OtOpcUa — the cookie stores the - /// directly. Because no internal - /// bearer validation path exists, the short "Role" key is intentionally used here rather - /// than the long URI; external consumers receive exactly the - /// key they expect. - /// - /// - /// If a JwtBearer scheme is ever added: the - /// passed to - /// AddJwtBearer MUST set RoleClaimType = JwtTokenService.RoleClaimType (and - /// NameClaimType = JwtTokenService.UsernameClaimType) so that - /// [Authorize(Roles=...)] and ClaimsPrincipal.IsInRole resolve correctly. - /// is already wired to do this and MUST be used - /// rather than constructing - /// ad hoc. + /// The short "Role" key is what mints and what external consumers of the token + /// expect. The internal bearer scheme (AddJwtBearer in AddOtOpcUaAuth) validates with + /// , which sets RoleClaimType = RoleClaimType and + /// NameClaimType = UsernameClaimType, and turns MapInboundClaims off, so + /// [Authorize(Roles=...)] and ClaimsPrincipal.IsInRole resolve against this key. Never + /// construct ad hoc. /// /// public const string RoleClaimType = "Role"; @@ -150,5 +138,9 @@ public sealed class JwtTokenService // if this is ever passed to AddJwtBearer. See RoleClaimType doc comment for rationale. RoleClaimType = RoleClaimType, NameClaimType = UsernameClaimType, + // Pinned to the one algorithm Issue signs with: an unsigned (alg:none) token or one signed with a + // different HMAC variant over the same key is refused on both the bearer and TryValidate paths. + ValidAlgorithms = [SecurityAlgorithms.HmacSha256], + RequireSignedTokens = true, }; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs index d58fbcf3..494145be 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.Cookies; +using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; @@ -108,8 +109,21 @@ public static class ServiceCollectionExtensions }); if (jwtConfigured) { - authBuilder.AddScheme( - OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { }); + // First-party JwtBearer (R03:2.4, fix round 1 / I1) — the earlier hand-rolled handler is gone. + // Validation parameters come from the SAME JwtTokenService that mints, so issuer/audience/key/ + // claim types can never drift; BuildValidationParameters pins ValidAlgorithms = HS256 (an + // alg:none or a different-HMAC token is refused). MapInboundClaims=false keeps the minted + // "Role"/"zb:username" claim keys verbatim, which RoleClaimType/NameClaimType name. A bad token + // is AuthenticateResult.Fail ⇒ 401 + WWW-Authenticate: Bearer, never a 500. + authBuilder.AddJwtBearer(OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { }); + services.AddOptions(OtOpcUaAuthSchemes.Bearer) + .Configure((o, jwt) => + { + o.TokenValidationParameters = jwt.BuildValidationParameters(); + o.MapInboundClaims = false; + o.SaveToken = false; + o.IncludeErrorDetails = false; + }); } if (disableLogin) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj index d81e4eb6..507aca6c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj @@ -10,6 +10,7 @@ + diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs index e5e3b620..599d0cf2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs @@ -12,6 +12,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; using Shouldly; using Xunit; using ZB.MOM.WW.Auth.AspNetCore; @@ -208,6 +209,98 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); } + /// + /// Fix round 1 / I1 — the first-party JwtBearer handler pins HS256: an unsigned (alg: none) + /// token carrying the right issuer/audience/roles, and a token signed with a different HMAC + /// variant, both fail CLOSED with a 401 — never a 500, never a pass. + /// + [Theory] + [InlineData("none")] + [InlineData("HS384")] + public async Task Bearer_with_a_non_HS256_algorithm_is_rejected_with_401(string alg) + { + var token = alg == "none" ? UnsignedToken() : SignedToken(SecurityAlgorithms.HmacSha384); + var client = NewClient(); + var req = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + + var response = await client.SendAsync(req, Ct); + + response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized); + response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); + + // Positive control: the same claims signed HS256 with the configured key ARE accepted, so the + // 401 above measured the algorithm and not a broken token shape. + var good = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + good.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue( + "Bearer", SignedToken(SecurityAlgorithms.HmacSha256)); + (await client.SendAsync(good, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + } + + /// + /// Fix round 1 / I2 — /auth/token mints only for a COOKIE session. A bearer token used to + /// be able to mint a fresh token for itself, so a stolen token could renew itself forever and + /// outlive both its own expiry and the user's LDAP membership. Positive control: the cookie + /// session that minted the bearer still gets a token. + /// + [Fact] + public async Task Auth_token_refuses_a_bearer_only_caller_and_serves_a_cookie_session() + { + var client = NewClient(); + var loginResponse = await client.PostAsJsonAsync("/auth/login", + new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + loginResponse.EnsureSuccessStatusCode(); + + var cookieReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + AttachCookies(cookieReq, loginResponse); + var cookieResp = await client.SendAsync(cookieReq, Ct); + cookieResp.StatusCode.ShouldBe(HttpStatusCode.OK); + var bearer = (await cookieResp.Content.ReadFromJsonAsync(Ct)).GetProperty("token").GetString()!; + + // The bearer authenticates elsewhere (positive control on the token itself) … + var root = new HttpRequestMessage(HttpMethod.Get, "/"); + root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer); + (await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + // … but cannot renew itself. + var renew = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + renew.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer); + renew.Headers.Add("X-Requested-With", "XMLHttpRequest"); + var renewResp = await client.SendAsync(renew, Ct); + renewResp.StatusCode.ShouldBeOneOf(HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden); + (await renewResp.Content.ReadAsStringAsync(Ct)).ShouldNotContain("token"); + } + + private const string TestSigningKey = "test-signing-key-with-at-least-32-bytes-of-utf8-content"; + + private static IEnumerable ViewerClaims() => + [ + new(JwtTokenService.UsernameClaimType, "mallory"), + new(JwtTokenService.RoleClaimType, "Viewer"), + ]; + + /// A token signed with using the configured key bytes. + private static string SignedToken(string algorithm) + { + var keyBytes = System.Text.Encoding.UTF8.GetBytes(TestSigningKey); + // The SAME key bytes for every algorithm, so a rejection measures the algorithm pin alone. + var token = new System.IdentityModel.Tokens.Jwt.JwtSecurityToken( + issuer: "otopcua-test", audience: "otopcua-test", claims: ViewerClaims(), + expires: DateTime.UtcNow.AddMinutes(5), + signingCredentials: new SigningCredentials(new SymmetricSecurityKey(keyBytes), algorithm)); + return new System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler().WriteToken(token); + } + + /// An alg: none token with otherwise valid issuer/audience/lifetime/roles. + private static string UnsignedToken() + { + static string B64(string json) => Microsoft.IdentityModel.Tokens.Base64UrlEncoder.Encode(json); + var exp = DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeSeconds(); + return B64("{\"alg\":\"none\",\"typ\":\"JWT\"}") + "." + + B64($"{{\"zb:username\":\"mallory\",\"Role\":\"Viewer\",\"iss\":\"otopcua-test\",\"aud\":\"otopcua-test\",\"exp\":{exp}}}") + + "."; + } + private async Task LoginAndIssueTokenAsync() { var client = NewClient(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs index 96fc88ca..9d2cc7ef 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs @@ -65,7 +65,7 @@ public sealed class JwtConfigurationGateTests var schemes = host.Services.GetRequiredService(); var bearer = await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer); bearer.ShouldNotBeNull(); - bearer.HandlerType.ShouldBe(typeof(JwtBearerAuthenticationHandler)); + bearer.HandlerType.ShouldBe(typeof(Microsoft.AspNetCore.Authentication.JwtBearer.JwtBearerHandler)); await host.StopAsync(Ct); } From c0a0e7aa81626b5ff31e4a75284ac0aa5205334d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:36:51 -0400 Subject: [PATCH 11/19] fix(security): audit actor falls back to HttpContext when the server auth-state provider has no circuit ServerAuthenticationStateProvider throws InvalidOperationException from GetAuthenticationStateAsync in any scope where no circuit set its state (minimal-API requests, hosted-service scopes). The accessor now treats that as "no circuit identity" and falls back to the HTTP principal. Pinned with the real provider on both sides (unset -> HTTP fallback; set -> circuit principal wins). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AuthenticationStateAuditActorAccessor.cs | 32 +++++++++++++---- ...henticationStateAuditActorAccessorTests.cs | 36 +++++++++++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs index d689b85e..13ede448 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs @@ -47,17 +47,37 @@ public sealed class AuthenticationStateAuditActorAccessor : IAuditActorAccessor { if (_authenticationState is not null) { - var task = _authenticationState.GetAuthenticationStateAsync(); - if (task.IsCompletedSuccessfully) - { - var actor = ResolveActor(task.Result.User); - if (actor is not null) return actor; - } + var actor = TryResolveCircuitActor(_authenticationState); + if (actor is not null) return actor; } return ResolveActor(_httpContextAccessor.HttpContext?.User); } } + /// + /// The circuit's actor, or null when this scope has no circuit identity. The server-side provider + /// is registered for EVERY scope on an AdminUI node, but only a circuit calls + /// SetAuthenticationState; anywhere else (a minimal-API request, a hosted service's scope) + /// GetAuthenticationStateAsync throws synchronously + /// (fix round 1 / I5). That is "no circuit", never an audit failure — the caller falls back to + /// the HTTP context. + /// + /// The scoped provider. + /// The actor, or null. + private static string? TryResolveCircuitActor(AuthenticationStateProvider provider) + { + Task task; + try + { + task = provider.GetAuthenticationStateAsync(); + } + catch (InvalidOperationException) + { + return null; + } + return task.IsCompletedSuccessfully ? ResolveActor(task.Result.User) : null; + } + /// The claim-resolution rule shared with . /// The principal, or null. /// The actor string, or null when unauthenticated. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs index 5dc1f391..6341ef6f 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs @@ -58,6 +58,42 @@ public sealed class AuthenticationStateAuditActorAccessorTests accessor.CurrentActor.ShouldBeNull(); } + /// + /// Fix round 1 / I5 — the REAL server-side provider, resolved in a non-circuit scope (a minimal-API + /// request, a background service), throws from + /// GetAuthenticationStateAsync because nothing called SetAuthenticationState. That used + /// to escape CurrentActor; it must fall back to the HTTP principal instead. + /// + [Fact] + public void Real_server_provider_outside_a_circuit_falls_back_to_the_http_principal() + { + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "dave")) }; + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(context), + new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider()); + + accessor.CurrentActor.ShouldBe("dave"); + + // No HTTP principal either ⇒ null, still no throw. + new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider()) + .CurrentActor.ShouldBeNull(); + } + + /// Positive control on the real provider: once a circuit has set its state, that principal wins. + [Fact] + public void Real_server_provider_inside_a_circuit_serves_the_circuit_principal() + { + var provider = new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider(); + provider.SetAuthenticationState(Task.FromResult( + new AuthenticationState(Authenticated(new Claim(ZbClaimTypes.Username, "erin"))))); + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "http-user")) }; + + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), provider) + .CurrentActor.ShouldBe("erin"); + } + private static ClaimsPrincipal Authenticated(params Claim[] claims) => new(new ClaimsIdentity(claims, "TestScheme", ZbClaimTypes.Name, ZbClaimTypes.Role)); From 88268ea3a6980a0c473d18b3acc8740add65d426 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:38:03 -0400 Subject: [PATCH 12/19] fix(host): OPC UA endpoint probe is Ready-only on a fused admin+driver node Coordinator ruling (I6): until ServiceLevel consumes IOpcUaEndpointState (F-C1), the probe must not sit on the Active tier of a fused node, because Traefik routes the AdminUI on /health/active. Driver-only nodes keep it on Ready + Active; admin-only nodes do not register it. Pinned per role. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Health/HealthEndpoints.cs | 21 ++++++++++++---- .../Health/OpcUaEndpointHealthCheck.cs | 6 +++-- .../Health/ActiveTierRegistrationTests.cs | 25 ++++++++++++------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs index 2f57158f..824223dd 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs @@ -31,7 +31,8 @@ public static class HealthEndpoints /// /// /// Whether this node carries the driver role and therefore serves an OPC UA endpoint. When - /// true the joins the ready + active tiers (R03:2.1), reading + /// true the joins the ready tier, and the active tier on a + /// driver-ONLY node (R03:2.1; on a fused node the active tier routes the AdminUI — see the body), reading /// the the host registers alongside OtOpcUaServerHostedService. /// Defaults to false so admin-only callers and the existing harnesses register nothing new. /// @@ -40,16 +41,26 @@ public static class HealthEndpoints { var checks = services.AddHealthChecks(); - // A driver node's whole purpose is its OPC UA endpoint, so a failed SDK start must take it out of - // the ready AND active tiers: readiness because it cannot serve, active because a load balancer or - // the fleet must not route to it. Factory-registered so the state singleton is resolved per probe. + // A failed SDK start takes the node out of the READY tier on every driver node: it cannot serve. + // The ACTIVE tier depends on the roles (coordinator ruling, fix round 1 / I6): + // - driver-only: Active too — a site node's whole purpose is its OPC UA endpoint, so nothing may + // route to it; + // - fused admin+driver: NOT Active, until the ServiceLevel calculator consumes IOpcUaEndpointState + // (follow-up F-C1). Traefik routes the AdminUI on /health/active, so putting the probe there would + // take the operator's only console out of rotation precisely when they need it to diagnose the + // failed endpoint, while the OPC UA failover (ServiceLevel) still would not move. Once F-C1 lands + // the peer takes Primary and the active tier follows it on its own. + // Factory-registered so the state singleton is resolved per probe. if (hasDriver) { + var tags = hasAdmin + ? new[] { ZbHealthTags.Ready } + : new[] { ZbHealthTags.Ready, ZbHealthTags.Active }; checks.Add(new HealthCheckRegistration( OpcUaEndpointHealthCheck.Name, sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService()), failureStatus: null, - tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })); + tags: tags)); } // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs index 8caa69f5..e619d7b3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs @@ -4,14 +4,16 @@ using ZB.MOM.WW.OtOpcUa.Host.OpcUa; namespace ZB.MOM.WW.OtOpcUa.Host.Health; /// -/// Ready + Active tier probe for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is +/// Ready-tier probe (plus Active on a driver-only node) for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is /// listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy /// one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets /// connection-refused. /// /// /// Registered on driver-role nodes only (); an -/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. +/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. On a fused +/// admin+driver node it is Ready-only until ServiceLevel consumes the fact (F-C1), because the Active tier +/// is what Traefik routes the AdminUI on. /// public sealed class OpcUaEndpointHealthCheck : IHealthCheck { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs index d880e7b3..45ff262b 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs @@ -50,31 +50,38 @@ public sealed class ActiveTierRegistrationTests } /// - /// R03:2.1 — the OPC UA endpoint probe is on BOTH tiers of a driver node (it cannot serve, and must not - /// be routed to) and absent from an admin-only node, which starts no endpoint to be degraded by. + /// R03:2.1 + fix round 1 / I6 (coordinator ruling) — the OPC UA endpoint probe's tiers by role: + /// + /// driver-only: Ready + Active (it cannot serve and must not be routed to); + /// fused admin+driver: Ready ONLY, until the ServiceLevel calculator consumes the fact + /// (F-C1) — Traefik routes the AdminUI on /health/active, so a failed SDK on the active + /// central node must not take the AdminUI out of rotation; + /// admin-only: absent (it starts no endpoint to be degraded by). + /// /// [Theory] - [InlineData(true)] - [InlineData(false)] - public void Opcua_endpoint_probe_is_registered_on_ready_and_active_for_driver_nodes_only(bool hasDriver) + [InlineData(false, true, true, true)] + [InlineData(true, true, true, false)] + [InlineData(true, false, false, false)] + public void Opcua_endpoint_probe_tiers_follow_the_node_roles(bool hasAdmin, bool hasDriver, bool ready, bool active) { var registrations = new ServiceCollection() - .AddOtOpcUaHealth(hasAdmin: true, hasDriver: hasDriver) + .AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: hasDriver) .BuildServiceProvider() .GetRequiredService>() .Value.Registrations .Where(r => r.Name == OpcUaEndpointHealthCheck.Name) .ToList(); - if (!hasDriver) + if (!ready && !active) { registrations.ShouldBeEmpty(); return; } var registration = registrations.ShouldHaveSingleItem(); - registration.Tags.ShouldContain(ZbHealthTags.Ready); - registration.Tags.ShouldContain(ZbHealthTags.Active); + registration.Tags.Contains(ZbHealthTags.Ready).ShouldBe(ready); + registration.Tags.Contains(ZbHealthTags.Active).ShouldBe(active); } [Fact] From f1c52385edf42387e892a00dcad64bfb133a7838 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:39:34 -0400 Subject: [PATCH 13/19] fix(controlplane): retention keep window counts Sealed rows; the newest Sealed blob is never pruned The window used to count every terminal row, so 50+ failed deploys pushed the last good artifact out of it. Now the newest KeepSealed (>= 1) Sealed rows keep their blob, failed rows are prunable unless referenced in the ReferencedDays window. Pinned: 60 TimedOut + 1 old Sealed -> Sealed retained, 60 TimedOut emptied. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Retention/DeploymentRetentionActor.cs | 21 ++++-- .../Retention/DeploymentRetentionOptions.cs | 7 +- .../DeploymentRetentionTests.cs | 65 ++++++++++++++++++- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs index 55ffd7dc..67020803 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs @@ -97,8 +97,9 @@ public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers } /// - /// The retention rule. Empties the blob of every terminal deployment that is (a) older than the - /// newest terminal rows, (b) not referenced by a + /// The retention rule. Empties the blob of every terminal deployment that is (a) not one of the + /// newest SEALED rows (at least one — the newest + /// Sealed blob is never pruned), (b) not referenced by a /// NodeDeploymentState row started or applied within /// , and (c) still carrying bytes — at most /// per call. @@ -125,11 +126,23 @@ public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers || d.Status == DeploymentStatus.PartiallyFailed || d.Status == DeploymentStatus.TimedOut) .OrderByDescending(d => d.CreatedAtUtc) - .Select(d => new { d.DeploymentId, HasBlob = d.ArtifactBlob.Length > 0 }) + .Select(d => new { d.DeploymentId, d.Status, HasBlob = d.ArtifactBlob.Length > 0 }) .ToListAsync(ct) .ConfigureAwait(false); - var aged = terminal.Skip(keep).Where(d => d.HasBlob).Select(d => d.DeploymentId).ToList(); + // The keep window counts SEALED rows only (fix round 1 / I3). Counting every terminal row let a burst + // of failed deploys (TimedOut / PartiallyFailed) fill the window by itself and push the fleet's last + // good artifact — the newest Sealed row, which boot, drift detection and the publish fallback all + // read — out of it. `keep` is clamped to ≥ 1, so the newest Sealed row is never a candidate. + var keptSealed = terminal + .Where(d => d.Status == DeploymentStatus.Sealed) + .Take(keep) + .Select(d => d.DeploymentId) + .ToHashSet(); + var aged = terminal + .Where(d => d.HasBlob && !keptSealed.Contains(d.DeploymentId)) + .Select(d => d.DeploymentId) + .ToList(); if (aged.Count == 0) return 0; var referenced = (await db.NodeDeploymentStates diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs index 6654ec1e..6bdc571f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs @@ -17,9 +17,10 @@ public sealed class DeploymentRetentionOptions public bool Enabled { get; set; } = true; /// - /// How many of the most recent terminal (Sealed / PartiallyFailed / TimedOut) deployments keep their - /// blob regardless of age. A node booting from central SQL reads the LATEST sealed row's blob, so this - /// must stay well above 1; 50 also keeps a comfortable re-deploy history. + /// How many of the most recent SEALED deployments keep their blob regardless of age (clamped to ≥ 1). + /// Failed deploys (PartiallyFailed / TimedOut) do not count toward it, so a burst of them can never push + /// the last good artifact out. A node booting from central SQL reads the LATEST sealed row's blob, so + /// this must stay well above 1; 50 also keeps a comfortable re-deploy history. /// public int KeepSealed { get; set; } = 50; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs index ecd397c4..bb607aaf 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs @@ -24,7 +24,7 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase public async Task Sweep_empties_blobs_beyond_the_keep_window_and_leaves_the_newest_intact() { var f = NewInMemoryDbFactory(); - var ids = Seed(f, sealedCount: 12); // ids[0] newest … ids[11] oldest + var ids = Seed(f, sealedCount: 12, mixInTimedOut: false); // ids[0] newest … ids[11] oldest var options = new DeploymentRetentionOptions { KeepSealed = 5, ReferencedDays = 30, MaxRowsPerSweep = 200 }; int pruned; @@ -91,6 +91,65 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase (await verify.Deployments.SingleAsync(d => d.DeploymentId == inFlight)).ArtifactBlob.ShouldNotBeEmpty("in flight"); } + /// + /// Fix round 1 / I3 — the keep window counts SEALED rows, not "terminal" ones. A burst of 60 failed + /// deploys used to fill a 50-row terminal window on its own and push the fleet's last good (Sealed) + /// artifact out of it — the one every boot / drift / fallback path reads. The newest Sealed blob is + /// never pruned; the unreferenced TimedOut rows are (positive control: the sweep did run). + /// + [Fact] + public async Task Sixty_timed_out_deploys_never_prune_the_newest_sealed_blob() + { + var f = NewInMemoryDbFactory(); + var sealedId = Guid.NewGuid(); + var timedOut = new List(); + await using (var db = f.CreateDbContext()) + { + db.Deployments.Add(new Deployment + { + DeploymentId = sealedId, RevisionHash = new string('s', 64), Status = DeploymentStatus.Sealed, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-200), SealedAtUtc = SweepNow.AddDays(-200), + ArtifactBlob = new byte[] { 1, 2, 3 }, + }); + for (var i = 0; i < 60; i++) + { + var id = Guid.NewGuid(); + timedOut.Add(id); + db.Deployments.Add(new Deployment + { + DeploymentId = id, RevisionHash = new string('t', 64), Status = DeploymentStatus.TimedOut, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddHours(-i), ArtifactBlob = new byte[] { 9 }, + }); + } + await db.SaveChangesAsync(); + } + var options = new DeploymentRetentionOptions { KeepSealed = 50, ReferencedDays = 30, MaxRowsPerSweep = 200 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(60); + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == sealedId)).ArtifactBlob + .ShouldNotBeEmpty("the newest Sealed artifact is never pruned"); + (await verify.Deployments.CountAsync(d => timedOut.Contains(d.DeploymentId) && d.ArtifactBlob.Length == 0)) + .ShouldBe(60); + } + + /// Even with KeepSealed misconfigured to 0, the newest Sealed row keeps its blob. + [Fact] + public async Task The_newest_sealed_blob_survives_a_zero_keep_window() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 3, mixInTimedOut: false); + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, new DeploymentRetentionOptions { KeepSealed = 0 }, SweepNow)) + .ShouldBe(2); + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty(); + } + [Fact] public async Task Sweep_is_bounded_per_call_and_drains_a_backlog_over_calls() { @@ -149,7 +208,7 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase } /// Seeds sealed deployments, newest first; each one hour older than the previous. - private static List Seed(IDbContextFactory f, int sealedCount) + private static List Seed(IDbContextFactory f, int sealedCount, bool mixInTimedOut = true) { var ids = new List(); using var db = f.CreateDbContext(); @@ -161,7 +220,7 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase { DeploymentId = id, RevisionHash = new string('a', 64), - Status = i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed, + Status = mixInTimedOut && i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed, CreatedBy = "test", CreatedAtUtc = SweepNow.AddHours(-i), SealedAtUtc = SweepNow.AddHours(-i), From f7e7ba0263045b1ce0ae1bbba3a60b3ead7bc12a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:42:29 -0400 Subject: [PATCH 14/19] fix(controlplane): audit writer drops only already-persisted EventIds, never the whole SQL Server batch SQL Server's DbUpdateException for one duplicate names every entry in the statement batch, so detaching ex.Entries dropped the whole flush. On a row-level rejection the writer now asks the database which of the batch's EventIds it already holds, drops exactly those, re-saves the rest, and falls back to row-by-row only if a non-duplicate row is still refused. Fresh context + entities per attempt. Pinned against the real central SQL Server (new SqlServerTestDatabase harness) and a faithful whole-batch interceptor. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Audit/AuditWriterActor.cs | 134 +++++++++++------- .../AuditWriterActorRejectedRowTests.cs | 109 ++++++++++++-- .../Harness/SqlServerTestDatabase.cs | 77 ++++++++++ 3 files changed, 255 insertions(+), 65 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs index 0e20da2f..3f0aa914 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs @@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; /// Dedup is two-layer: in-buffer (the below collapses /// duplicate EventIds before flush) and at the database via the filtered unique index /// UX_ConfigAuditLog_EventId (cross-restart safety — a retry of an already-flushed -/// batch hits the constraint; detaches the rejected row -/// and re-saves, so the duplicate is dropped without losing the rest of the batch). +/// batch hits the constraint; drops exactly the already-persisted EventIds +/// and re-saves the rest, so the duplicate is dropped without losing the rest of the batch). /// /// Implements the shared seam: is a /// best-effort, never-throwing entry point that simply Tells this actor and returns @@ -92,22 +92,7 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter try { - using var db = _dbFactory.CreateDbContext(); - foreach (var evt in snapshot) - { - db.ConfigAuditLogs.Add(new ConfigAuditLog - { - Timestamp = evt.OccurredAtUtc.UtcDateTime, - Principal = evt.Actor, - EventType = $"{evt.Category}:{evt.Action}", - NodeId = evt.SourceNode, - DetailsJson = evt.DetailsJson, - EventId = evt.EventId, - CorrelationId = evt.CorrelationId, - Outcome = evt.Outcome.ToString(), - }); - } - var dropped = SaveDetachingRejectedRows(db); + var dropped = SaveBatch(snapshot); _log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped); } catch (Exception ex) @@ -116,46 +101,93 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter } } + private static ConfigAuditLog ToRow(AuditEvent evt) => new() + { + Timestamp = evt.OccurredAtUtc.UtcDateTime, + Principal = evt.Actor, + EventType = $"{evt.Category}:{evt.Action}", + NodeId = evt.SourceNode, + DetailsJson = evt.DetailsJson, + EventId = evt.EventId, + CorrelationId = evt.CorrelationId, + Outcome = evt.Outcome.ToString(), + }; + /// - /// Saves the batch, detaching and retrying past any row the database rejects (R03:2.6). One - /// statement batch used to carry the whole flush: a single duplicate EventId (any at-least-once - /// producer — the audit seam is Tell-based through a singleton proxy that re-delivers across a - /// hand-over) threw for all 500 rows and the class remarks' per-row dedup claim was not what ran. + /// Persists one flush, dropping ONLY the rows the database will not take (R03:2.6, fix round 1 / I4). /// - /// The context holding the batch. + /// The flush. /// How many rows were dropped. /// - /// Only a that NAMES its entries is handled here — that is the shape a - /// constraint violation takes (SQL 2601/2627 for the unique index). A failure with no entries (the - /// connection died, a deadlock) is the whole-batch failure it always was and propagates to the - /// caller's catch. Bounded by the batch size so a store that rejects every row still terminates. + /// + /// Why not detach ex.Entries. SQL Server inserts the batch as one statement batch, so + /// the raised for ONE duplicate EventId names EVERY entry in + /// the batch. Detaching what it names dropped the whole flush — the exact loss the fix was for + /// (pinned against a real SQL Server by On_sql_server_a_duplicate_event_id_costs_only_that_row). + /// + /// + /// So, on a rejection that names entries (a constraint violation; 2601/2627 for the unique index): + /// (1) ask the database which of the batch's EventIds it already holds and drop exactly those — + /// the at-least-once re-delivery the dedup contract expects; (2) re-save the rest as a batch; (3) if + /// that is still rejected (some OTHER per-row constraint), fall back to row-by-row and drop only the + /// rows that fail on their own. Every attempt uses a fresh context and fresh entities, so no state + /// from a failed save leaks into the next one. A failure that names no entry (connection loss, + /// deadlock) propagates — the whole-batch failure it always was. Bounded: at most 2 + N saves. + /// /// - private int SaveDetachingRejectedRows(OtOpcUaConfigDbContext db) + private int SaveBatch(IReadOnlyList batch) { - var dropped = 0; - var attempts = 0; - var maxAttempts = db.ChangeTracker.Entries().Count() + 1; - while (true) + if (TrySave(batch, out var first)) return 0; + _log.Debug(first, "AuditWriter: a batch of {Count} was rejected; dropping only already-persisted EventIds", batch.Count); + + HashSet existing; + using (var db = _dbFactory.CreateDbContext()) { - try - { - db.SaveChanges(); - return dropped; - } - catch (DbUpdateException ex) when (ex.Entries.Count > 0 && ++attempts < maxAttempts) - { - var duplicate = IsUniqueViolation(ex); - foreach (var entry in ex.Entries) - { - var eventId = (entry.Entity as ConfigAuditLog)?.EventId; - if (duplicate) - _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", eventId); - else - _log.Warning(ex, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", eventId); - entry.State = EntityState.Detached; - dropped++; - } - } + var ids = batch.Select(e => (Guid?)e.EventId).ToList(); + existing = db.ConfigAuditLogs + .AsNoTracking() + .Where(r => ids.Contains(r.EventId)) + .Select(r => r.EventId!.Value) + .ToHashSet(); + } + foreach (var id in existing) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", id); + + var remaining = batch.Where(e => !existing.Contains(e.EventId)).ToList(); + var dropped = batch.Count - remaining.Count; + if (remaining.Count == 0 || TrySave(remaining, out _)) return dropped; + + // Still rejected with no duplicate left to blame: isolate the offending row(s). + foreach (var evt in remaining) + { + if (TrySave([evt], out var rowError)) continue; + dropped++; + if (IsUniqueViolation(rowError!)) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", evt.EventId); + else + _log.Warning(rowError, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", evt.EventId); + } + return dropped; + } + + /// Saves on a fresh context; a rejection that names entries is reported, anything else throws. + /// The rows to save. + /// The row-level rejection, when this returns false. + /// when saved. + private bool TrySave(IReadOnlyList events, out DbUpdateException? rejection) + { + using var db = _dbFactory.CreateDbContext(); + foreach (var evt in events) db.ConfigAuditLogs.Add(ToRow(evt)); + try + { + db.SaveChanges(); + rejection = null; + return true; + } + catch (DbUpdateException ex) when (ex.Entries.Count > 0 || IsUniqueViolation(ex)) + { + rejection = ex; + return false; } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs index bd64161a..aba37648 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs @@ -26,7 +26,7 @@ public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase var dbName = Guid.NewGuid().ToString("N"); var root = new InMemoryDatabaseRoot(); var duplicate = Guid.NewGuid(); - var rejecting = new RejectingDbFactory(dbName, root, new RejectOnce(duplicate)); + var rejecting = new RejectingDbFactory(dbName, root, new AlwaysRejectBatchContaining(duplicate)); var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); @@ -46,6 +46,68 @@ public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase persisted.ShouldContain(ids[8]); } + /// + /// Fix round 1 / I4 — a FAITHFUL SQL Server shape: the provider sends the batch as one statement batch, + /// so the for one duplicate names EVERY entry in it. Detaching + /// ex.Entries therefore dropped the whole flush. The writer must drop only the rows whose + /// EventId is already persisted and keep the rest. + /// + [Fact] + public void A_whole_batch_rejection_drops_only_the_rows_already_persisted() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var factory = new RejectingDbFactory(dbName, root, new RejectWholeBatchOnDuplicate()); + var duplicate = Guid.NewGuid(); + using (var seed = factory.CreateDbContext()) + { + seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate }); + seed.SaveChanges(); + } + var actor = Sys.ActorOf(AuditWriterActor.Props(factory)); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor); + + using var db = factory.CreateDbContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(21, "the 20 new rows land beside the one that was already there"); + persisted.Count(id => id == duplicate).ShouldBe(1); + db.ConfigAuditLogs.Single(r => r.EventId == duplicate).Principal.ShouldBe("earlier"); + } + + /// The same contract against the REAL provider and the real filtered unique index. + [Fact] + [Trait("Category", "SchemaCompliance")] + public void On_sql_server_a_duplicate_event_id_costs_only_that_row() + { + using var sql = new SqlServerTestDatabase(); + var duplicate = Guid.NewGuid(); + using (var seed = sql.NewContext()) + { + seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate }); + seed.SaveChanges(); + } + var actor = Sys.ActorOf(AuditWriterActor.Props(sql.Factory())); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor, PresenceBudget); + + using var db = sql.NewContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(21); + persisted.Count(id => id == duplicate).ShouldBe(1); + foreach (var id in ids) persisted.ShouldContain(id); + } + /// A failure that names no entry is a whole-batch failure, exactly as before. [Fact] public void A_connection_level_failure_still_drops_the_batch_without_looping() @@ -71,24 +133,43 @@ public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(), }; - /// Throws a naming the entry for one EventId, the first time it is saved. - private sealed class RejectOnce(Guid eventId) : ISaveChangesInterceptor + /// + /// A row the database refuses on its own terms (not a duplicate — e.g. a per-row constraint), every + /// time it is in the save. Faithful to SQL Server's batch shape: the exception names EVERY added + /// entry, so only a row-by-row retry can isolate the offender. + /// + private sealed class AlwaysRejectBatchContaining(Guid eventId) : ISaveChangesInterceptor { - private int _fired; - public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) { - if (_fired == 1 || eventData.Context is null) return result; - var entry = eventData.Context.ChangeTracker.Entries() - .FirstOrDefault(e => e.Entity.EventId == eventId); - if (entry is null) return result; - _fired = 1; - // GetInfrastructure() hands back the pubternal InternalEntityEntry, which is the IUpdateEntry the - // real provider puts on the exception; the test only needs that one cast. + if (eventData.Context is null) return result; + var added = eventData.Context.ChangeTracker.Entries() + .Where(e => e.State == EntityState.Added).ToList(); + if (!added.Any(e => e.Entity.EventId == eventId)) return result; #pragma warning disable EF1001 - var updateEntry = (IUpdateEntry)entry.GetInfrastructure(); + var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList(); #pragma warning restore EF1001 - throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601)", new[] { updateEntry }); + throw new DbUpdateException("The statement conflicted with a CHECK constraint (simulated, whole batch)", entries); + } + } + + /// + /// Models SQL Server's batched insert: when ANY added row's EventId is already stored, the whole save + /// fails with an exception naming EVERY added entry (the InMemory provider enforces no unique index, + /// so this interceptor is the constraint). + /// + private sealed class RejectWholeBatchOnDuplicate : ISaveChangesInterceptor + { + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) + { + if (eventData.Context is not OtOpcUaConfigDbContext ctx) return result; + var added = ctx.ChangeTracker.Entries().Where(e => e.State == EntityState.Added).ToList(); + var ids = added.Select(e => e.Entity.EventId).Where(id => id != null).ToList(); + if (ids.Count == 0 || !ctx.ConfigAuditLogs.AsNoTracking().Any(r => ids.Contains(r.EventId))) return result; +#pragma warning disable EF1001 + var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList(); +#pragma warning restore EF1001 + throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601, whole batch)", entries); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs new file mode 100644 index 00000000..3415bb99 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs @@ -0,0 +1,77 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +/// +/// A migrated, throw-away config database on the central SQL Server, for the few ControlPlane behaviours +/// the InMemory provider cannot model (a SQL Server DbUpdateException names the WHOLE statement +/// batch; collation-ordered reads). Same server / credentials / env overrides as +/// Configuration.Tests.SchemaComplianceFixture (OTOPCUA_CONFIG_TEST_SERVER, +/// OTOPCUA_CONFIG_TEST_SA_PASSWORD; default the shared docker host's dev SA), and the same database +/// name shape, so that suite's 24 h orphan reaper also reclaims a database a killed run leaks here. +/// +public sealed class SqlServerTestDatabase : IDisposable +{ + private const string DefaultServer = "10.100.0.35,14330"; + private const string DefaultSaPassword = "OtOpcUaDev_2026!"; + + /// Gets the connection string of the migrated database. + public string ConnectionString { get; } + + private readonly string _databaseName; + + /// Creates and migrates the database. + public SqlServerTestDatabase() + { + var server = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SERVER") ?? DefaultServer; + var saPassword = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SA_PASSWORD") ?? DefaultSaPassword; + _databaseName = $"OtOpcUaConfig_Test_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}"; + ConnectionString = + $"Server={server};Database={_databaseName};User Id=sa;Password={saPassword};TrustServerCertificate=True;Encrypt=False;"; + using var ctx = NewContext(); + ctx.Database.Migrate(); + } + + /// A new context on the database. + /// The context. + public OtOpcUaConfigDbContext NewContext() => + new(new DbContextOptionsBuilder().UseSqlServer(ConnectionString).Options); + + /// A factory over . + /// The factory. + public IDbContextFactory Factory() => new ContextFactory(this); + + /// Drops the database (best effort; a leak is reaped by the Configuration suite). + public void Dispose() + { + try + { + SqlConnection.ClearAllPools(); + var master = new SqlConnectionStringBuilder(ConnectionString) { InitialCatalog = "master" }.ConnectionString; + using var conn = new SqlConnection(master); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = @" +DECLARE @sql nvarchar(max); +IF DB_ID(@name) IS NOT NULL +BEGIN + SET @sql = N'ALTER DATABASE ' + QUOTENAME(@name) + N' SET SINGLE_USER WITH ROLLBACK IMMEDIATE; ' + + N'DROP DATABASE ' + QUOTENAME(@name) + N';'; + EXEC sp_executesql @sql; +END"; + cmd.Parameters.Add(new SqlParameter("@name", _databaseName)); + cmd.ExecuteNonQuery(); + } + catch (Exception) + { + // Best effort — see the class remarks. + } + } + + private sealed class ContextFactory(SqlServerTestDatabase owner) : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => owner.NewContext(); + } +} From 9466b8b242f80d36f3ef6c481a6058e9eee46722 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:46:54 -0400 Subject: [PATCH 15/19] fix(controlplane): close a lost dispatch from the in-flight check; guard terminal rows; blob-free status writes I7: a Dispatching row committed after the coordinator's PreStart whose Tell never arrived had no owner and refused every later deploy. The in-flight check (now fully off the admin singleton's thread, serialised by the actor-local compose flag alone) closes a Dispatching row with zero NodeDeploymentState rows that is past the apply deadline as TimedOut (optimistic on RowVersion) and proceeds. HandleDispatch ignores a dispatch for a Sealed/PartiallyFailed/TimedOut row, so a late or duplicated dispatch cannot flip it back. Coordinator status/reason writes attach a key+RowVersion stub instead of loading the multi-MB ArtifactBlob; proven on the real SQL Server (blob untouched, seal lands). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdminOperations/AdminOperationsActor.cs | 172 +++++++++++++----- .../Coordinators/ConfigPublishCoordinator.cs | 84 +++++++-- .../ConfigPublishCoordinatorRecoveryTests.cs | 135 ++++++++++++++ 3 files changed, 332 insertions(+), 59 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index e78c926e..81f09dae 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -11,6 +11,7 @@ using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Configuration.Validation; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Scripting; @@ -28,6 +29,12 @@ public sealed class AdminOperationsActor : ReceiveActor private readonly IReadOnlyDictionary _probesByType; private readonly TagConfigValidationMode _tagConfigValidationMode; + /// + /// The coordinator's apply deadline. A Dispatching row with no expected-ack rows that is older + /// than this is a lost dispatch, closed by the in-flight check (fix round 1 / I7). + /// + private readonly TimeSpan _applyDeadline; + /// /// Where central→node commands are handed off (per-cluster mesh), or /// to publish on the mediator directly. @@ -48,15 +55,17 @@ public sealed class AdminOperationsActor : ReceiveActor /// Central comm actor commands are handed to, or to publish on the /// DistributedPubSub mediator directly (the behaviour before the per-cluster mesh). /// + /// The coordinator's apply deadline; defaults to . /// Props configured to create an AdminOperationsActor. public static Props Props( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, - IActorRef? meshRouter = null) => + IActorRef? meshRouter = null, + TimeSpan? applyDeadline = null) => Akka.Actor.Props.Create(() => - new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter)); + new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter, applyDeadline)); /// Initializes a new instance of the AdminOperationsActor. /// Factory for creating config database contexts. @@ -64,13 +73,16 @@ public sealed class AdminOperationsActor : ReceiveActor /// Driver probes registered in DI; keyed by DriverType (case-insensitive). /// Deploy-gate TagConfig strictness mode (Warn default | Error opt-in). /// Central comm actor, or for the mediator. + /// The coordinator's apply deadline; defaults to . public AdminOperationsActor( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, - IActorRef? meshRouter = null) + IActorRef? meshRouter = null, + TimeSpan? applyDeadline = null) { + _applyDeadline = applyDeadline ?? ConfigPublishCoordinator.DefaultApplyDeadline; _dbFactory = dbFactory; _coordinator = coordinator; _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase); @@ -201,16 +213,17 @@ public sealed class AdminOperationsActor : ReceiveActor } /// - /// Serialised part of StartDeployment: the in-flight check. Everything after it — the - /// snapshot read, validation, gzip and the row write — runs on the thread pool and comes back as a - /// / message, so a large compose no - /// longer suspends the mailbox (R03:3.1) and alarm acks keep flowing while it runs. + /// StartDeployment. The only serialised part is the actor-local + /// flag; the DB in-flight check, the snapshot read, validation, gzip and the row write all run on the + /// thread pool and come back as a / + /// message, so a slow or blocked ConfigDb never suspends the mailbox (R03:3.1; fix round 1 moved the + /// in-flight query off the actor thread too) and alarm acks keep flowing while it runs. /// /// - /// is what keeps the check correct once the compose is off-actor: the - /// Deployment row is written at the END of the compose, so a second request arriving mid-compose - /// would pass a DB-only check and two artifacts would race for one coordinator. The flag is cleared - /// when the outcome message lands, on both branches. + /// is what keeps the check correct once it is off-actor: it is set + /// BEFORE the off-actor work starts and cleared when the outcome message lands (both branches), so a + /// second request is refused in memory for the whole check-plus-compose and two artifacts can never + /// race for one coordinator. /// private void HandleStartDeployment(StartDeployment msg) { @@ -226,49 +239,118 @@ public sealed class AdminOperationsActor : ReceiveActor return; } - try - { - // Refuse if any deployment is already in flight — keeps the coordinator's state - // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. One indexed - // projection (IX_Deployment_Status), synchronous on purpose: it IS the serialised part. - using var db = _dbFactory.CreateDbContext(); - var inflight = db.Deployments - .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) - .Select(d => d.DeploymentId) - .FirstOrDefault(); - if (inflight != Guid.Empty) - { - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.AnotherDeploymentInFlight, - DeploymentId: new DeploymentId(inflight), - RevisionHash: null, - Message: $"Deployment {inflight:N} is still in flight.", - msg.CorrelationId)); - return; - } - } - catch (Exception ex) - { - _log.Error(ex, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.CreatedBy, msg.CorrelationId); - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Rejected, - DeploymentId: null, - RevisionHash: null, - Message: ServerFault("StartDeployment", msg.CorrelationId), - msg.CorrelationId)); - return; - } - _composeInFlight = msg; // Captured here: Context/Sender are not available on the thread-pool thread. var sourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown"; var self = Self; - Task.Run(() => ComposeDeploymentAsync(msg, sourceNode)) + Task.Run(async () => + { + var refusal = await CheckInFlightAsync(msg).ConfigureAwait(false); + return refusal ?? await ComposeDeploymentAsync(msg, sourceNode).ConfigureAwait(false); + }) .PipeTo(self, success: composed => new DeployComposed(msg, replyTo, composed.Result, composed.Dispatch), failure: ex => new DeployComposeFailed(msg, replyTo, ex)); } + /// + /// The DB half of the in-flight check, off the actor thread. Refuses while a deployment is in flight — + /// keeping the coordinator's state unambiguous — EXCEPT a lost dispatch, which it closes first. + /// + /// The request. + /// The refusal, or null to proceed. + /// + /// + /// Lost dispatch (fix round 1 / I7). A Dispatching row with ZERO + /// NodeDeploymentState rows was never taken by the coordinator (its HandleDispatch + /// seeds the expected set and flips the status in one SaveChanges). The coordinator's PreStart + /// recovers such a row — but only one that already existed when it started. A row committed AFTER + /// PreStart whose Tell never arrived (this actor died between the compose's commit and + /// ) has no owner at all, and refused every later deploy forever. + /// Once it is past the apply deadline nothing can still be legitimately dispatching it, so it is + /// closed TimedOut with a reason and the new deploy proceeds. A late DispatchDeployment + /// for it is ignored by the coordinator's terminal-state guard. + /// + /// + /// One indexed projection (IX_Deployment_Status) plus, only for a Dispatching row past the + /// deadline, a count and a tracked update. + /// + /// + private async Task CheckInFlightAsync(StartDeployment msg) + { + await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false); + var inflight = await db.Deployments + .AsNoTracking() + .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) + .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, d.Status, d.CreatedAtUtc }) + .ToListAsync() + .ConfigureAwait(false); + + foreach (var row in inflight) + { + if (row.Status == DeploymentStatus.Dispatching + && DateTime.UtcNow - row.CreatedAtUtc > _applyDeadline + && !await db.NodeDeploymentStates.AnyAsync(s => s.DeploymentId == row.DeploymentId).ConfigureAwait(false) + && await CloseLostDispatchAsync(db, row.DeploymentId).ConfigureAwait(false)) + { + continue; + } + + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.AnotherDeploymentInFlight, + DeploymentId: new DeploymentId(row.DeploymentId), + RevisionHash: null, + Message: $"Deployment {row.DeploymentId:N} is still in flight.", + msg.CorrelationId), + Dispatch: null); + } + return null; + } + + /// Closes a lost dispatch as TimedOut, unless it moved on meanwhile (optimistic, on its RowVersion). + /// The context. + /// The row. + /// when the row is now closed. + private async Task CloseLostDispatchAsync(OtOpcUaConfigDbContext db, Guid deploymentId) + { + var stamp = await db.Deployments + .AsNoTracking() + .Where(d => d.DeploymentId == deploymentId && d.Status == DeploymentStatus.Dispatching) + .Select(d => new { d.RowVersion }) + .FirstOrDefaultAsync() + .ConfigureAwait(false); + if (stamp is null) return true; // no longer Dispatching — not in flight as far as this check goes + + // Stub update: the row's ArtifactBlob is never read just to change its status. + var stub = new Deployment + { + DeploymentId = deploymentId, RevisionHash = string.Empty, CreatedBy = string.Empty, + RowVersion = stamp.RowVersion, + Status = DeploymentStatus.TimedOut, + FailureReason = + "Dispatch was lost before any node was expected to ack (the coordinator was never told, or never " + + "took it) and the apply deadline has passed; closed by the next StartDeployment. Redeploy.", + }; + var entry = db.Deployments.Attach(stub); + entry.Property(d => d.Status).IsModified = true; + entry.Property(d => d.FailureReason).IsModified = true; + try + { + await db.SaveChangesAsync().ConfigureAwait(false); + } + catch (DbUpdateConcurrencyException) + { + // The coordinator took it after all (status flipped, RowVersion moved): it is genuinely in flight. + return false; + } + _log.Warning( + "StartDeployment: closed lost dispatch {DeploymentId} as TimedOut — Dispatching with no expected-ack rows " + + "past the {Deadline} apply deadline (the coordinator never took it)", deploymentId, _applyDeadline); + return true; + } + private void HandleDeployComposed(DeployComposed msg) { _composeInFlight = null; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index 3c99e998..49b54df4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -234,6 +234,26 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers private void HandleDispatch(DispatchDeployment msg) { + // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already + // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message + // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it. + using (var check = _dbFactory.CreateDbContext()) + { + var status = check.Deployments + .AsNoTracking() + .Where(d => d.DeploymentId == msg.DeploymentId.Value) + .Select(d => (DeploymentStatus?)d.Status) + .FirstOrDefault(); + if (status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) + { + _log.Warning( + "Ignoring DispatchDeployment {Id}: the deployment is already {Status} (a late or duplicated dispatch)", + msg.DeploymentId, status); + if (_current == msg.DeploymentId) ResetForNext(); + return; + } + } + _current = msg.DeploymentId; _acks.Clear(); _degraded.Clear(); @@ -371,11 +391,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // same as clean, though: the deployment's reason names the degraded nodes (per-node detail is on // each NodeDeploymentState row) and the seal logs at Warning. var nodes = _degraded.Select(n => n.Value).Order(StringComparer.OrdinalIgnoreCase).ToList(); - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); - if (d is not null) - d.FailureReason = Truncate( - $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " + - "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it."); + SetFailureReason(db, Truncate( + $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " + + "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it.")); db.SaveChanges(); _log.Warning( "Deployment {Id} sealed DEGRADED (acks={Count}) — the address-space apply was degraded on: {Nodes}", @@ -420,11 +438,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut); if (_expectedAcks.Count == 0) { - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); - if (d is not null) - d.FailureReason = Truncate( - "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + - "hand-over mid-dispatch) and the apply deadline has passed. Redeploy."); + SetFailureReason(db, Truncate( + "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + + "hand-over mid-dispatch) and the apply deadline has passed. Redeploy.")); db.SaveChanges(); _log.Warning( "Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " + @@ -459,13 +475,53 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers public sealed record DeadlineElapsed(DeploymentId DeploymentId); + /// + /// Stages a status change WITHOUT materialising the row: the row carries the whole ArtifactBlob + /// (varbinary(max), multi-MB on a real fleet), which no status transition reads (fix round 1). A stub + /// carrying only the key and the row's current RowVersion is attached and the changed columns marked + /// modified, so the UPDATE writes those columns and keeps the optimistic-concurrency check. The staged + /// stub stays tracked so can add to the same UPDATE. + /// private static void UpdateDeploymentStatus( OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false) { - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == id.Value); - if (d is null) return; - d.Status = status; - if (sealNow) d.SealedAtUtc = DateTime.UtcNow; + var entry = TrackStub(db, id); + if (entry is null) return; + entry.Entity.Status = status; + entry.Property(x => x.Status).IsModified = true; + if (sealNow) + { + entry.Entity.SealedAtUtc = DateTime.UtcNow; + entry.Property(x => x.SealedAtUtc).IsModified = true; + } + } + + /// Stages the current deployment's failure reason onto the stub staged. + private void SetFailureReason(OtOpcUaConfigDbContext db, string reason) + { + if (_current is null) return; + var entry = TrackStub(db, _current.Value); + if (entry is null) return; + entry.Entity.FailureReason = reason; + entry.Property(x => x.FailureReason).IsModified = true; + } + + /// The tracked blob-free stub for a deployment, attaching one on first use; null when the row is gone. + private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry? TrackStub( + OtOpcUaConfigDbContext db, DeploymentId id) + { + var tracked = db.ChangeTracker.Entries().FirstOrDefault(e => e.Entity.DeploymentId == id.Value); + if (tracked is not null) return tracked; + var stamp = db.Deployments + .AsNoTracking() + .Where(x => x.DeploymentId == id.Value) + .Select(x => new { x.RowVersion }) + .FirstOrDefault(); + if (stamp is null) return null; + return db.Deployments.Attach(new Deployment + { + DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = stamp.RowVersion, + }); } /// diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs index 86f6efa7..0added3d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -176,6 +176,141 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes }, duration: PresenceBudget); } + /// + /// Fix round 1 / I7 — the window PreStart cannot see: the Dispatching row is committed AFTER the + /// coordinator's PreStart ran, and the coordinator is never told (the admin singleton died between + /// its commit and the Tell). Nothing would ever close it. The in-flight check now closes a + /// Dispatching row with NO expected-ack rows once it is past the apply deadline, so the next + /// StartDeployment is Accepted (and seals — positive control on the pipeline). + /// + [Fact] + public async Task Lost_dispatch_committed_after_PreStart_is_closed_by_the_next_StartDeployment() + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, "site-a-1:4053"); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart saw nothing + + var stale = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + dbFactory, coordinator, Enumerable.Empty(), applyDeadline: NoTimeout)); + + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var result = ExpectMsg(PresenceBudget); + result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, result.Message); + + using (var db = dbFactory.CreateDbContext()) + { + var row = db.Deployments.Single(d => d.DeploymentId == stale.Value); + row.Status.ShouldBe(DeploymentStatus.TimedOut); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("lost"); + } + + coordinator.Tell(new ApplyAck(result.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == result.DeploymentId.Value.Value).Status + .ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// Control for the closure: the same row INSIDE the deadline is still in flight and still refuses. + [Fact] + public void Dispatching_row_inside_the_deadline_still_refuses_a_new_deployment() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe(); + var pending = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(-5)); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + dbFactory, coordinator.Ref, Enumerable.Empty(), applyDeadline: NoTimeout)); + + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var result = ExpectMsg(PresenceBudget); + result.Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight); + result.DeploymentId.ShouldBe(pending); + + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching); + } + + /// + /// Fix round 1 minor — a late or duplicated DispatchDeployment for a row that is already + /// terminal (it raced the lost-dispatch closure above, or a re-delivery after a seal) must not flip + /// it back to AwaitingApplyAcks or seed an expected set. Positive control: a live row dispatched right + /// after it IS seeded (so the first message was processed, not merely not-yet-processed). + /// + [Theory] + [InlineData(DeploymentStatus.Sealed)] + [InlineData(DeploymentStatus.TimedOut)] + [InlineData(DeploymentStatus.PartiallyFailed)] + public void Dispatch_of_a_terminal_row_is_ignored(DeploymentStatus terminal) + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, "site-a-1:4053"); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + var done = SeedDeployment(dbFactory, terminal, DateTime.UtcNow.AddMinutes(-1)); + coordinator.Tell(new DispatchDeployment(done, TestRevision, CorrelationId.NewId())); + var live = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == live.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + using var verify = dbFactory.CreateDbContext(); + verify.Deployments.Single(d => d.DeploymentId == done.Value).Status.ShouldBe(terminal); + verify.NodeDeploymentStates.Count(s => s.DeploymentId == done.Value).ShouldBe(0); + } + + /// + /// Fix round 1 minor — status transitions update a blob-free stub (key + RowVersion) instead of + /// materialising the multi-MB row. Proven on the REAL provider, where the stub's RowVersion is the + /// optimistic-concurrency predicate: dispatch → ack → Sealed lands, and the artifact bytes are + /// untouched by the partial UPDATEs. + /// + [Fact] + [Trait("Category", "SchemaCompliance")] + public void On_sql_server_status_transitions_do_not_touch_the_artifact_blob() + { + using var sql = new SqlServerTestDatabase(); + var factory = sql.Factory(); + SeedNodes(factory, "site-a-1:4053"); + var blob = Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray(); + var id = DeploymentId.NewId(); + using (var db = sql.NewContext()) + { + db.Deployments.Add(new Deployment + { + DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = DeploymentStatus.Dispatching, + CreatedBy = "test", ArtifactBlob = blob, + }); + db.SaveChanges(); + } + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(factory, NoTimeout)); + AwaitAssert(() => + { + using var db = sql.NewContext(); + db.Deployments.AsNoTracking().Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); // PreStart re-drives the un-dispatched row + coordinator.Tell(new ApplyAck(id, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = sql.NewContext(); + var row = db.Deployments.AsNoTracking().Single(); + row.Status.ShouldBe(DeploymentStatus.Sealed); + row.SealedAtUtc.ShouldNotBeNull(); + row.ArtifactBlob.ShouldBe(blob); + }, duration: PresenceBudget); + } + private static DeploymentId SeedDeployment( IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) { From 2adeacc65582d4704cbc254c4c6e4194058c7d4a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:48:40 -0400 Subject: [PATCH 16/19] fix(controlplane): keep the revision hash byte-identical to the pre-refactor composer The one-snapshot refactor re-sorted every collection in memory with an ordinal comparer, while the old composer's SQL ORDER BY used the database's case-insensitive collation; mixed-case keys therefore re-hashed an unchanged configuration (spurious drift fleet-wide, a "new" revision on a no-op redeploy). The snapshot factory now reads every composed table ORDER BY its natural key server-side and Compose serialises in the order given. Pinned byte-for-byte against a verbatim replica of the old composer on the real SQL Server. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Validation/DraftSnapshotFactory.cs | 40 +++++---- .../AdminOperations/ConfigComposer.cs | 51 ++++++----- ...nfigComposerSqlServerHashStabilityTests.cs | 87 +++++++++++++++++++ 3 files changed, 139 insertions(+), 39 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs index 154d664c..30873aca 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs @@ -56,6 +56,12 @@ public static class DraftSnapshotFactory } } + // Every composed table is read ORDERED BY ITS NATURAL KEY, server-side. That order IS the deployment + // artifact's order (ConfigComposer.Compose serialises the snapshot as given): it is exactly the order the + // pre-R03:3.2 composer's own queries produced — the database collation's, case-insensitive on SQL Server — + // so the revision hash of an unchanged configuration is byte-identical across the refactor (fix round 1; + // pinned by ConfigComposerSqlServerHashStabilityTests). Re-sorting in memory with any other comparer would + // re-hash every mixed-case configuration and make drift detection report the whole fleet drifted. private static async Task ReadAsync(OtOpcUaConfigDbContext db, CancellationToken ct) => new DraftSnapshot { @@ -64,26 +70,26 @@ public static class DraftSnapshotFactory // EquipmentUuid checks run in separate validator passes — no rule here reads these fields. GenerationId = 0, // generation model dropped; placeholder (no rule reads it) ClusterId = string.Empty, // global snapshot; rules compare entity ClusterId fields, not this - ServerClusters = await db.ServerClusters.AsNoTracking().ToListAsync(ct), - ClusterNodes = await db.ClusterNodes.AsNoTracking().ToListAsync(ct), + ServerClusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), + ClusterNodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), // v3: Namespace entity retired — no namespace list to materialize. The Raw tree // (RawFolders/TagGroups) + UnsTagReferences feed the v3 rules (charset, tagname length, // effective-leaf uniqueness). - RawFolders = await db.RawFolders.AsNoTracking().ToListAsync(ct), - DriverInstances = await db.DriverInstances.AsNoTracking().ToListAsync(ct), - Devices = await db.Devices.AsNoTracking().ToListAsync(ct), - TagGroups = await db.TagGroups.AsNoTracking().ToListAsync(ct), - UnsAreas = await db.UnsAreas.AsNoTracking().ToListAsync(ct), - UnsLines = await db.UnsLines.AsNoTracking().ToListAsync(ct), - Equipment = await db.Equipment.AsNoTracking().ToListAsync(ct), - Tags = await db.Tags.AsNoTracking().ToListAsync(ct), - UnsTagReferences = await db.UnsTagReferences.AsNoTracking().ToListAsync(ct), - VirtualTags = await db.VirtualTags.AsNoTracking().ToListAsync(ct), - ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().ToListAsync(ct), - Scripts = await db.Scripts.AsNoTracking().ToListAsync(ct), - PollGroups = await db.PollGroups.AsNoTracking().ToListAsync(ct), - NodeAcls = await db.NodeAcls.AsNoTracking().ToListAsync(ct), - ApiKeys = await db.ApiKeys.AsNoTracking().ToListAsync(ct), + RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), + DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), + Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), + TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), + UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), + UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), + Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), + Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), + UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), + VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), + ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), + PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), + NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), PriorEquipment = [], // intentional: no prior-generation table to diff against at this level ActiveReservations = await db.ExternalIdReservations .AsNoTracking() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs index 6f332143..39735ddb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs @@ -9,8 +9,9 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; /// /// Pure snapshot composer: reads the current live-edit state from /// and serialises it into a deterministic, gzip-compressed byte[] artifact + SHA-256 hex revision hash. -/// Determinism comes from sorting every collection by its natural key before serialising, so two snapshots -/// over the same DB state always produce the same hash regardless of EF row ordering; compression wraps the +/// Determinism comes from reading every collection ordered by its natural key on the server +/// (), so two snapshots over the same DB state always produce the same hash +/// — and the same hash as the pre-R03:3.2 composer, which used those same queries; compression wraps the /// final bytes and reorders nothing. The hash is taken over the COMPRESSED bytes — see . /// public static class ConfigComposer @@ -41,6 +42,11 @@ public static class ConfigComposer /// Serialises an already-read into the deterministic artifact. Pure over /// its input: the same snapshot always yields the same bytes and hash, whatever the DB holds by now. /// + /// + /// Serialises the collections IN THE ORDER GIVEN. Determinism comes from + /// , which reads every table ordered by its natural key + /// on the server; a snapshot built any other way must be ordered the same way or its hash will differ. + /// /// The rows to compose — the SAME rows the validator was run over. /// The compressed artifact blob and its revision hash. public static ConfigArtifact Compose(DraftSnapshot snapshot) @@ -52,34 +58,35 @@ public static class ConfigComposer // projection (UnsAreas/UnsLines/Equipment/UnsTagReferences) and the retained VirtualTags/ScriptedAlarms/ // PollGroups/NodeAcls/Scripts. The Namespaces snapshot is RETIRED (the two OPC UA namespaces are implicit). // RevisionHash inputs follow the blob automatically — adding/removing a table shifts the SHA-256. - // Every collection is sorted by its natural key with an ORDINAL comparer so the hash is independent of - // both EF row order and the process culture (a culture-aware sort could order "a-1"/"A-1" differently - // on two central nodes and make the same config hash twice). + // NO re-sort here (fix round 1): the snapshot's collections arrive ordered by their natural key from + // DraftSnapshotFactory's server-side ORDER BY — the same order, and so the same bytes and RevisionHash, + // as the pre-R03:3.2 composer's own queries. An in-memory ordinal re-sort orders mixed-case keys + // differently from the SQL collation and silently re-hashed every unchanged configuration. var ordered = new { - Clusters = snapshot.ServerClusters.OrderBy(x => x.ClusterId, StringComparer.Ordinal).ToList(), - Nodes = snapshot.ClusterNodes.OrderBy(x => x.NodeId, StringComparer.Ordinal).ToList(), - RawFolders = snapshot.RawFolders.OrderBy(x => x.RawFolderId, StringComparer.Ordinal).ToList(), - DriverInstances = snapshot.DriverInstances.OrderBy(x => x.DriverInstanceId, StringComparer.Ordinal).ToList(), - Devices = snapshot.Devices.OrderBy(x => x.DeviceId, StringComparer.Ordinal).ToList(), - TagGroups = snapshot.TagGroups.OrderBy(x => x.TagGroupId, StringComparer.Ordinal).ToList(), - Equipment = snapshot.Equipment.OrderBy(x => x.EquipmentId, StringComparer.Ordinal).ToList(), - Tags = snapshot.Tags.OrderBy(x => x.TagId, StringComparer.Ordinal).ToList(), - UnsTagReferences = snapshot.UnsTagReferences.OrderBy(x => x.UnsTagReferenceId, StringComparer.Ordinal).ToList(), - PollGroups = snapshot.PollGroups.OrderBy(x => x.PollGroupId, StringComparer.Ordinal).ToList(), - UnsAreas = snapshot.UnsAreas.OrderBy(x => x.UnsAreaId, StringComparer.Ordinal).ToList(), - UnsLines = snapshot.UnsLines.OrderBy(x => x.UnsLineId, StringComparer.Ordinal).ToList(), - NodeAcls = snapshot.NodeAcls.OrderBy(x => x.NodeAclId, StringComparer.Ordinal).ToList(), + Clusters = snapshot.ServerClusters, + Nodes = snapshot.ClusterNodes, + RawFolders = snapshot.RawFolders, + DriverInstances = snapshot.DriverInstances, + Devices = snapshot.Devices, + TagGroups = snapshot.TagGroups, + Equipment = snapshot.Equipment, + Tags = snapshot.Tags, + UnsTagReferences = snapshot.UnsTagReferences, + PollGroups = snapshot.PollGroups, + UnsAreas = snapshot.UnsAreas, + UnsLines = snapshot.UnsLines, + NodeAcls = snapshot.NodeAcls, // the data-plane credentials travel with the config, because a driver-only node // has no ConfigDb to read them from (per-cluster mesh Phase 4) and would otherwise deny every // UserName token. Only the peppered HMAC of each secret ships — never the secret — and the // pepper is node configuration that is deliberately NOT in the artifact, so a stolen blob // still cannot be turned into a working credential. This is also why revoking a key reaches a // driver-only node only on the next deployment; see the ApiKey entity's remarks. - ApiKeys = snapshot.ApiKeys.OrderBy(x => x.ApiKeyId, StringComparer.Ordinal).ToList(), - Scripts = snapshot.Scripts.OrderBy(x => x.ScriptId, StringComparer.Ordinal).ToList(), - VirtualTags = snapshot.VirtualTags.OrderBy(x => x.VirtualTagId, StringComparer.Ordinal).ToList(), - ScriptedAlarms = snapshot.ScriptedAlarms.OrderBy(x => x.ScriptedAlarmId, StringComparer.Ordinal).ToList(), + ApiKeys = snapshot.ApiKeys, + Scripts = snapshot.Scripts, + VirtualTags = snapshot.VirtualTags, + ScriptedAlarms = snapshot.ScriptedAlarms, }; return Pack(JsonSerializer.SerializeToUtf8Bytes(ordered, JsonOptions)); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs new file mode 100644 index 00000000..67ac26bd --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs @@ -0,0 +1,87 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Fix round 1 minor — revision-hash stability across the R03:3.2 refactor. The pre-change composer read +/// every table with a SQL ORDER BY (the database's case-INSENSITIVE collation); the refactor re-sorted +/// in memory with an ORDINAL comparer, which orders mixed-case keys differently ("B-1" before "a-2"). The +/// same configuration would then hash differently after the upgrade: drift detection reports every node +/// drifted, and a re-deploy of an unchanged config shows a "new" revision. This pins the composer's output +/// BYTE-identical to a verbatim replica of the pre-change composer on the real provider. +/// +[Trait("Category", "SchemaCompliance")] +public sealed class ConfigComposerSqlServerHashStabilityTests +{ + [Fact] + public async Task Composer_output_is_byte_identical_to_the_pre_change_composer_for_mixed_case_keys() + { + using var sql = new SqlServerTestDatabase(); + await using (var seed = sql.NewContext()) + { + // Collation order a-2, B-1, c-3; ordinal order B-1, a-2, c-3. + foreach (var id in new[] { "c-3", "B-1", "a-2" }) + seed.Scripts.Add(new Script { ScriptId = id, Name = id, SourceCode = "return 1;", SourceHash = id }); + await seed.SaveChangesAsync(); + } + + await using var legacyDb = sql.NewContext(); + var legacy = await LegacySnapshotAndFlattenAsync(legacyDb); + + await using var viaDb = sql.NewContext(); + var current = await ConfigComposer.SnapshotAndFlattenAsync(viaDb); + current.Blob.ShouldBe(legacy.Blob); + current.RevisionHash.ShouldBe(legacy.RevisionHash); + + // The deploy path's form (validate + compose from ONE snapshot) must agree too. + await using var deployDb = sql.NewContext(); + var viaSnapshot = ConfigComposer.Compose(await DraftSnapshotFactory.FromConfigDbAsync(deployDb)); + viaSnapshot.RevisionHash.ShouldBe(legacy.RevisionHash); + } + + /// Verbatim replica of ConfigComposer.SnapshotAndFlattenAsync before R03:3.2 (ad2b5c40). + private static async Task<(byte[] Blob, string RevisionHash)> LegacySnapshotAndFlattenAsync(OtOpcUaConfigDbContext db) + { + var ct = CancellationToken.None; + var snapshot = new + { + Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), + Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), + RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), + DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), + Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), + TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), + Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), + Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), + UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), + PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), + UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), + UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), + NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), + Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), + VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), + ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + }; + var json = JsonSerializer.SerializeToUtf8Bytes(snapshot, new JsonSerializerOptions + { + WriteIndented = false, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never, + }); + using var output = new MemoryStream(json.Length / 8); + using (var gzip = new GZipStream(output, CompressionLevel.Fastest, leaveOpen: true)) + gzip.Write(json, 0, json.Length); + var blob = output.ToArray(); + return (blob, Convert.ToHexStringLower(SHA256.HashData(blob))); + } +} From 896aaa2b02258e4f829f7d5fdff10c7efb57e0b0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:49:56 -0400 Subject: [PATCH 17/19] fix(controlplane): a PSK secret deleted from the store revokes the key; bound refresh concurrency RefreshAsync now tells "the store answered: no such secret" (revocation - drop the refreshed key AND the lazily cached one) apart from "the read failed" (keep the previous key). Store reads fan out through Parallel.ForEachAsync capped at MaxConcurrentRefreshReads = 8 instead of one read per cluster at once. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Communication/IMeshPskProvider.cs | 90 +++++++++++++------ .../MeshPskProviderRefreshTests.cs | 71 +++++++++++++++ 2 files changed, 136 insertions(+), 25 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs index bdab7ba7..bad1956c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs @@ -110,7 +110,8 @@ public interface IMeshPskProvider /// /// Re-reads the keys for from the store OFF the request path and /// swaps them into the cache atomically (R03:3.4). Until the refresh completes, lookups keep - /// serving the previous key; a cluster whose read fails keeps its previous key too. Replaces + /// serving the previous key; a cluster whose read FAILS keeps its previous key too, while a cluster whose + /// secret the store reports ABSENT is revoked (no key from then on). Replaces /// "invalidate on every refresh", which made the first inbound ack per cluster per /// ContactRefreshSeconds block a gRPC thread on a synchronous, 3 s-bounded store read. /// @@ -331,45 +332,76 @@ public sealed class MeshPskProvider : IMeshPskProvider _refreshed = new Dictionary(StringComparer.OrdinalIgnoreCase); } + /// + /// Ceiling on concurrent store reads during one . A fleet of N clusters used to + /// fan out N simultaneous reads every contact refresh; the store (and anything behind it) sees at most + /// this many at once now. + /// + public const int MaxConcurrentRefreshReads = 8; + /// public async Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) { ArgumentNullException.ThrowIfNull(clusterIds); var previous = _refreshed; - var next = new Dictionary(StringComparer.OrdinalIgnoreCase); var ids = clusterIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + var results = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); - // Resolve every cluster concurrently — each read bounded like the synchronous path — and carry a - // failed cluster's PREVIOUS refreshed key forward: a transient store fault must not turn into a - // window in which that cluster has no key at all. - var reads = ids.Select(async id => + // Bounded fan-out; each read bounded like the synchronous path. + await Parallel.ForEachAsync( + ids, + new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRefreshReads, CancellationToken = ct }, + async (id, token) => results[id] = await ResolveFromStoreAsync(id, token).ConfigureAwait(false)) + .ConfigureAwait(false); + + // Three outcomes per cluster (fix round 1 / I8): + // - Found → the new key; + // - NotFound → the store answered that the secret does not exist: a REVOCATION — the cluster gets no + // key (and its lazily cached one is evicted below), so a deleted secret stops working + // within one refresh instead of living on for the life of the process; + // - Failed → the store could not be read (threw / timed out / unnameable): keep the PREVIOUS key — + // a transient fault must not open a window in which the cluster has no key at all. + var next = new Dictionary(StringComparer.OrdinalIgnoreCase); + var answered = new List(); + foreach (var id in ids) { - var value = await ResolveFromStoreAsync(id, ct).ConfigureAwait(false); - return (Id: id, Value: value); - }).ToList(); - foreach (var (id, value) in await Task.WhenAll(reads).ConfigureAwait(false)) - { - if (!string.IsNullOrWhiteSpace(value)) next[id] = value; - else if (previous.TryGetValue(id, out var kept)) next[id] = kept; + var read = results[id]; + switch (read.Outcome) + { + case StoreReadOutcome.Found: + next[id] = read.Value!; + answered.Add(id); + break; + case StoreReadOutcome.NotFound: + answered.Add(id); + break; + default: + if (previous.TryGetValue(id, out var kept)) next[id] = kept; + break; + } } - // The swap. Then drop the lazy entries for the clusters now covered, so a key rotated between - // refreshes is served from the new snapshot rather than from a stale lazy resolution; entries - // for clusters NOT covered (their read failed and they had no previous value) are left alone. + // The swap. Then drop the lazy entries for every cluster the store ANSWERED for — found (serve the new + // snapshot, not a stale lazy resolution) or not found (revoked: a lazily cached key must die too). + // Entries for clusters whose read failed are left alone. _refreshed = next; - foreach (var id in next.Keys) _cache.TryRemove(id, out _); + foreach (var id in answered) _cache.TryRemove(id, out _); } + private enum StoreReadOutcome { Found, NotFound, Failed } + + private readonly record struct StoreRead(StoreReadOutcome Outcome, string? Value); + /// - /// The asynchronous twin of : same name derivation, same bound, same - /// "anything but a usable key is null" contract — awaited instead of blocked on, because this one - /// runs on a refresh rather than inside a gRPC interceptor. + /// The asynchronous twin of : same name derivation, same bound — awaited + /// instead of blocked on, because this one runs on a refresh rather than inside a gRPC interceptor — + /// but it keeps "the secret does not exist" (a blank answer) apart from "the store could not be read". /// /// The application Cluster id. /// The caller's token, linked under the store-read bound. - /// The stored key, or null. - private async Task ResolveFromStoreAsync(string clusterId, CancellationToken ct) + /// The outcome and, when found, the key. + private async Task ResolveFromStoreAsync(string clusterId, CancellationToken ct) { SecretName name; try @@ -384,14 +416,22 @@ public sealed class MeshPskProvider : IMeshPskProvider + "refreshed from the store. Supply it through MeshTransport:ClusterPsks instead, or " + "rename the cluster to [a-z0-9._/-].", clusterId); - return null; + return new StoreRead(StoreReadOutcome.Failed, null); } using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); cts.CancelAfter(StoreReadTimeout); try { - return await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false); + var value = await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(value)) return new StoreRead(StoreReadOutcome.Found, value); + + _logger.LogWarning( + "Mesh PSK secret {SecretName} for cluster {ClusterId} is absent (or blank) in the store; the key is " + + "REVOKED — commands to that cluster and acks from it are refused until it is restored.", + name.Value, + clusterId); + return new StoreRead(StoreReadOutcome.NotFound, null); } catch (Exception ex) { @@ -401,7 +441,7 @@ public sealed class MeshPskProvider : IMeshPskProvider + "previously resolved key (if any) stays in service until the next refresh.", clusterId, name.Value); - return null; + return new StoreRead(StoreReadOutcome.Failed, null); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs index e727c153..34129235 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs @@ -78,6 +78,77 @@ public sealed class MeshPskProviderRefreshTests psk.ShouldBe("k-1"); } + /// + /// Fix round 1 / I8 — "the store says there is no such secret" is a REVOCATION, not a read failure: + /// the previous key must stop being served (from the refreshed snapshot AND the lazy cache). A thrown + /// read keeps the previous key (the test above); a definitive not-found drops it. + /// + [Fact] + public async Task A_secret_deleted_from_the_store_revokes_the_key_on_the_next_refresh() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + provider.Matches("SITE-A", "k-1").ShouldBeTrue("positive control: the key was in service"); + + resolver.Value = null; // the secret was deleted + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); + provider.Matches("SITE-A", "k-1").ShouldBeFalse(); + } + + /// The same revocation reaches a key that was first served from the LAZY cache, not a refresh. + [Fact] + public async Task A_deleted_secret_also_evicts_a_lazily_cached_key() + { + var resolver = new GatedResolver { Value = "k-lazy" }; + var provider = Provider(resolver); + provider.TryGetPsk("SITE-A", out var lazy).ShouldBeTrue(); + lazy.ShouldBe("k-lazy"); + + resolver.Value = null; + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); + } + + /// Fix round 1 minor — the refresh fans out to the store with BOUNDED concurrency, not one read per cluster at once. + [Fact] + public async Task Refresh_store_reads_are_concurrency_bounded() + { + var resolver = new ConcurrencyTrackingResolver(); + var provider = Provider(resolver); + var clusters = Enumerable.Range(0, 40).Select(i => $"SITE-{i}").ToList(); + + await provider.RefreshAsync(clusters, CancellationToken.None); + + resolver.Calls.ShouldBe(40); + resolver.MaxConcurrent.ShouldBeLessThanOrEqualTo(MeshPskProvider.MaxConcurrentRefreshReads); + resolver.MaxConcurrent.ShouldBeGreaterThan(1, "still concurrent, just bounded"); + foreach (var c in clusters) provider.TryGetPsk(c, out _).ShouldBeTrue(); + } + + private sealed class ConcurrencyTrackingResolver : ISecretResolver + { + private int _inFlight; + private int _max; + private int _calls; + public int MaxConcurrent => Volatile.Read(ref _max); + public int Calls => Volatile.Read(ref _calls); + + public async Task GetAsync(SecretName name, CancellationToken ct) + { + Interlocked.Increment(ref _calls); + var now = Interlocked.Increment(ref _inFlight); + int seen; + while (now > (seen = Volatile.Read(ref _max)) && Interlocked.CompareExchange(ref _max, now, seen) != seen) { } + await Task.Delay(20, ct); + Interlocked.Decrement(ref _inFlight); + return "k-" + name.Value; + } + } + /// A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour). [Fact] public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily() From dc3323fcca2d6a213fc63d5b1c95049015bd17ff Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:51:28 -0400 Subject: [PATCH 18/19] fix(host): publish the known-id set and its stamp atomically; never write the shared guard options BoundedApiKeyStore kept the set (volatile) and its DateTimeOffset stamp (non-volatile, 16 bytes) in separate fields, so a reader could pair a new set with an old stamp or read a torn stamp; they are now one immutable snapshot behind a volatile reference. An inner store that cannot list ids now switches the set off with a per-instance flag instead of writing KnownIdSet = false into the shared ApiKeyStoreGuardOptions. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../OpcUa/BoundedApiKeyStore.cs | 35 ++++++++++++------- .../OpcUa/BoundedApiKeyStoreKnownIdTests.cs | 30 ++++++++++++++++ 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index bb3154ed..6ebb7626 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -61,10 +61,22 @@ public sealed class BoundedApiKeyStore : IApiKeyStore // R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds // (single-flight; a refresh that fails or times out keeps the previous set). Null until the first // successful load, or when the inner store cannot list ids — then the per-id path runs as before. - private volatile IReadOnlySet? _knownIds; - private DateTimeOffset _knownIdsStampedUtc = DateTimeOffset.MinValue; + // The set and the instant it was loaded are published TOGETHER as one immutable snapshot behind a volatile + // reference (fix round 1): a reader on another thread can never pair a new set with an old stamp, or read + // a torn DateTimeOffset (a 16-byte struct, not atomically readable), as the separate fields allowed. + private volatile KnownIdSnapshot? _knownIds; private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1); + // Set when the inner store reports it cannot list ids. Per-instance on purpose: the guard options object is + // shared configuration and is never written (it used to be — `_guard.KnownIdSet = false` — which leaked + // one store's capability into every other consumer of the same options). + private volatile bool _listingUnsupported; + + /// The known-id set and the instant it was loaded, published as one unit. + /// The ids the inner store holds. + /// When they were read. + private sealed record KnownIdSnapshot(IReadOnlySet Ids, DateTimeOffset StampedUtc); + // R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards // the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the // configured rate instead of one SQL round trip per attempt under the SessionManager lock. @@ -104,7 +116,7 @@ public sealed class BoundedApiKeyStore : IApiKeyStore } /// The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards. - public IReadOnlySet? KnownKeyIds => _knownIds; + public IReadOnlySet? KnownKeyIds => _knownIds?.Ids; /// The store being bounded. /// @@ -173,26 +185,25 @@ public sealed class BoundedApiKeyStore : IApiKeyStore /// The set, or null. private async Task?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct) { - if (!_guard.KnownIdSet || ttl <= TimeSpan.Zero) return null; + if (!_guard.KnownIdSet || _listingUnsupported || ttl <= TimeSpan.Zero) return null; var now = _time.GetUtcNow(); var current = _knownIds; - if (current is not null && now - _knownIdsStampedUtc < ttl) return current; + if (current is not null && now - current.StampedUtc < ttl) return current.Ids; - if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current; + if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current?.Ids; try { // Re-check under the gate: another caller may have refreshed while this one waited. - if (_knownIds is { } refreshed && now - _knownIdsStampedUtc < ttl) return refreshed; + if (_knownIds is { } refreshed && now - refreshed.StampedUtc < ttl) return refreshed.Ids; var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false); if (ids is null) { - // Unsupported by the inner store: say so once and stop asking. - _guard.KnownIdSet = false; + // Unsupported by the inner store: stop asking — for this decorator only. + _listingUnsupported = true; return null; } - _knownIds = ids; - _knownIdsStampedUtc = _time.GetUtcNow(); + _knownIds = new KnownIdSnapshot(ids, _time.GetUtcNow()); return ids; } catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) @@ -200,7 +211,7 @@ public sealed class BoundedApiKeyStore : IApiKeyStore _log.LogWarning(ex, "ApiKeyStore: refreshing the known key-id set failed; {Mode}", current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force"); - return current; + return current?.Ids; } finally { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs index 7051b151..8d08df5e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs @@ -120,6 +120,36 @@ public sealed class BoundedApiKeyStoreKnownIdTests inner.ListCalls.ShouldBe(0); } + /// + /// Fix round 1 minor — an inner store that cannot list ids switches the known-id set off for THIS + /// decorator only. It used to do so by writing KnownIdSet = false into the guard options it was + /// handed, a shared options object, so one store's capability leaked into every other consumer. + /// Positive control: a second decorator over a listing store on the same options instance still uses + /// the set. + /// + [Fact] + public async Task An_unlisting_store_does_not_mutate_the_shared_guard_options() + { + var guard = new ApiKeyStoreGuardOptions { MissBurst = 100, MissRefillPerSecond = 20 }; + var options = Options.Create(new ApiKeyOptions + { + Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = 5, + }); + var clock = new FakeClock(T0); + var unlisting = new ListingStore("k-real") { SupportsListing = false }; + var first = new BoundedApiKeyStore(unlisting, options, NullLogger.Instance, clock, guard); + + (await first.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + unlisting.FindCalls.ShouldBe(1, "no set: the per-id path ran"); + guard.KnownIdSet.ShouldBeTrue("the shared options object is never written"); + + var listing = new ListingStore("k-real"); + var second = new BoundedApiKeyStore(listing, options, NullLogger.Instance, clock, guard); + (await second.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + listing.FindCalls.ShouldBe(0, "the second decorator still refuses unknown ids in memory"); + listing.ListCalls.ShouldBe(1); + } + private static BoundedApiKeyStore Bounded( IApiKeyStore inner, TimeProvider clock, int cacheSeconds, int missBurst = 100, double refillPerSecond = 20) => new( inner, From 5097444d56f9b7cad8ae979eba5c17fd35ed2c41 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:08:42 -0400 Subject: [PATCH 19/19] fix(controlplane): condition the dispatch flip on the guard's RowVersion; one deadline source; align IdentityModel pins Re-review 1: HandleDispatch's terminal guard and its AwaitingApplyAcks flip read the row separately, so a lost-dispatch closure committed between them was silently reopened. The flip now carries the RowVersion the guard read; a change in between is a concurrency conflict (nothing seeded, nothing published, coordinator reset). Pinned on real SQL Server with a closure injected between the two reads (fails without the fix). AdminOps' apply deadline and the coordinator's now both read DeployTimings.ApplyDeadline, passed explicitly to both singletons. Microsoft.IdentityModel.Protocols(.OpenIdConnect) pinned to 8.11.0 and referenced directly by the Security project, so the whole IdentityModel family resolves 8.11.0 (fresh restore clean of NU190x/NU1605). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- Directory.Packages.props | 8 ++- .../AdminOperations/AdminOperationsActor.cs | 9 +-- .../Coordinators/ConfigPublishCoordinator.cs | 63 +++++++++++----- .../ServiceCollectionExtensions.cs | 8 ++- .../ZB.MOM.WW.OtOpcUa.Security.csproj | 3 + .../ConfigPublishCoordinatorRecoveryTests.cs | 71 +++++++++++++++++++ 6 files changed, 138 insertions(+), 24 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index ed2dca67..d63e819e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -36,8 +36,10 @@ + line as the other Microsoft.AspNetCore.* pins. It depends on Microsoft.IdentityModel.Protocols(.OpenIdConnect) + 8.0.1; those two are pinned to 8.11.0 below (and referenced directly by the Security project, the JwtBearer + entry point, so the pin applies without transitive pinning) to keep the whole IdentityModel family on one + version with Microsoft.IdentityModel.Tokens / System.IdentityModel.Tokens.Jwt. --> @@ -86,6 +88,8 @@ + + diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index 81f09dae..69e878db 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -11,7 +11,6 @@ using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Configuration.Validation; -using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Scripting; @@ -55,7 +54,7 @@ public sealed class AdminOperationsActor : ReceiveActor /// Central comm actor commands are handed to, or to publish on the /// DistributedPubSub mediator directly (the behaviour before the per-cluster mesh). /// - /// The coordinator's apply deadline; defaults to . + /// The coordinator's apply deadline; defaults to (the coordinator's default source). /// Props configured to create an AdminOperationsActor. public static Props Props( IDbContextFactory dbFactory, @@ -73,7 +72,7 @@ public sealed class AdminOperationsActor : ReceiveActor /// Driver probes registered in DI; keyed by DriverType (case-insensitive). /// Deploy-gate TagConfig strictness mode (Warn default | Error opt-in). /// Central comm actor, or for the mediator. - /// The coordinator's apply deadline; defaults to . + /// The coordinator's apply deadline; defaults to (the coordinator's default source). public AdminOperationsActor( IDbContextFactory dbFactory, IActorRef coordinator, @@ -82,7 +81,9 @@ public sealed class AdminOperationsActor : ReceiveActor IActorRef? meshRouter = null, TimeSpan? applyDeadline = null) { - _applyDeadline = applyDeadline ?? ConfigPublishCoordinator.DefaultApplyDeadline; + // The same source the coordinator's default reads (ConfigPublishCoordinator.DefaultApplyDeadline IS + // DeployTimings.ApplyDeadline), so the lost-dispatch closure and the coordinator's own deadline agree. + _applyDeadline = applyDeadline ?? DeployTimings.ApplyDeadline; _dbFactory = dbFactory; _coordinator = coordinator; _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index 49b54df4..d408343c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -237,21 +237,28 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it. + // + // The RowVersion read HERE is the one the AwaitingApplyAcks flip below is conditioned on (re-review 1): + // reading a fresh one at flip time left a window in which AdminOps' lost-dispatch closure could commit + // TimedOut between the guard and the flip, and the flip would silently reopen the row. With the guard's + // RowVersion, any change since the guard read makes the flip a concurrency conflict instead. + byte[]? guardRowVersion; using (var check = _dbFactory.CreateDbContext()) { - var status = check.Deployments + var guard = check.Deployments .AsNoTracking() .Where(d => d.DeploymentId == msg.DeploymentId.Value) - .Select(d => (DeploymentStatus?)d.Status) + .Select(d => new { d.Status, d.RowVersion }) .FirstOrDefault(); - if (status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) + if (guard?.Status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) { _log.Warning( "Ignoring DispatchDeployment {Id}: the deployment is already {Status} (a late or duplicated dispatch)", - msg.DeploymentId, status); + msg.DeploymentId, guard.Status); if (_current == msg.DeploymentId) ResetForNext(); return; } + guardRowVersion = guard?.RowVersion; } _current = msg.DeploymentId; @@ -290,8 +297,22 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers }); } } - UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks); - db.SaveChanges(); + UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks, rowVersion: guardRowVersion); + try + { + db.SaveChanges(); + } + catch (DbUpdateConcurrencyException) + { + // The row moved after the guard read (on SQL Server: AdminOps closed it as a lost dispatch). The + // expected-set rows and the flip were one SaveChanges, so nothing was written; nothing is + // published and the coordinator is free for the next deployment. + _log.Warning( + "DispatchDeployment {Id} abandoned: the deployment changed state while it was being dispatched " + + "(closed as a lost dispatch?) — not reopening it", msg.DeploymentId); + ResetForNext(); + return; + } } // Per-cluster mesh Phase 2: hand off to the transport router rather than publishing @@ -483,9 +504,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers /// stub stays tracked so can add to the same UPDATE. /// private static void UpdateDeploymentStatus( - OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false) + OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false, byte[]? rowVersion = null) { - var entry = TrackStub(db, id); + var entry = TrackStub(db, id, rowVersion); if (entry is null) return; entry.Entity.Status = status; entry.Property(x => x.Status).IsModified = true; @@ -507,20 +528,30 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers } /// The tracked blob-free stub for a deployment, attaching one on first use; null when the row is gone. + /// The context. + /// The deployment. + /// + /// The RowVersion the update must be conditioned on, when the caller already read one (the dispatch + /// guard); reads the current one. + /// private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry? TrackStub( - OtOpcUaConfigDbContext db, DeploymentId id) + OtOpcUaConfigDbContext db, DeploymentId id, byte[]? rowVersion = null) { var tracked = db.ChangeTracker.Entries().FirstOrDefault(e => e.Entity.DeploymentId == id.Value); if (tracked is not null) return tracked; - var stamp = db.Deployments - .AsNoTracking() - .Where(x => x.DeploymentId == id.Value) - .Select(x => new { x.RowVersion }) - .FirstOrDefault(); - if (stamp is null) return null; + if (rowVersion is null) + { + var stamp = db.Deployments + .AsNoTracking() + .Where(x => x.DeploymentId == id.Value) + .Select(x => new { x.RowVersion }) + .FirstOrDefault(); + if (stamp is null) return null; + rowVersion = stamp.RowVersion; + } return db.Deployments.Attach(new Deployment { - DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = stamp.RowVersion, + DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = rowVersion, }); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index f838a004..9a9f0742 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; @@ -111,7 +112,7 @@ public static class ServiceCollectionExtensions // coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks // inside a props factory and races the spawn it is waiting for. var meshRouter = registry.Get(); - return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: meshRouter); + return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: DeployTimings.ApplyDeadline, meshRouter: meshRouter); }, singletonOptions); @@ -128,7 +129,10 @@ public static class ServiceCollectionExtensions resolver.GetService()?["Deployment:TagConfigValidationMode"], ignoreCase: true, out var m) ? m : TagConfigValidationMode.Warn; var meshRouter = registry.Get(); - return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter); + // The SAME deadline source as the coordinator above: AdminOps closes a lost dispatch only once it + // is past the coordinator's deadline, so the two must never drift. + return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter, + applyDeadline: DeployTimings.ApplyDeadline); }, singletonOptions); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj index 507aca6c..8ee03814 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj @@ -11,6 +11,9 @@ + + + diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs index 0added3d..8c93fb1c 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -311,6 +311,77 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes }, duration: PresenceBudget); } + /// + /// Re-review 1 — the reopen race. AdminOps' lost-dispatch closure commits TimedOut AFTER the coordinator's + /// terminal guard read the row as Dispatching but BEFORE the AwaitingApplyAcks flip. The flip is + /// conditioned on the guard's RowVersion, so on SQL Server it becomes a concurrency conflict and the row + /// stays TimedOut with no expected set. Positive control: the coordinator is not wedged — the next live + /// dispatch is seeded and flipped. + /// + [Fact] + [Trait("Category", "SchemaCompliance")] + public async Task On_sql_server_a_closure_between_guard_and_flip_is_not_reopened() + { + using var sql = new SqlServerTestDatabase(); + var inner = sql.Factory(); + SeedNodes(inner, "site-a-1:4053"); + var interposer = new InterposingDbFactory(inner); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(interposer, NoTimeout)); + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart ran over an empty table + + // The race proper: a new Dispatching row whose dispatch is interleaved with an AdminOps-style closure. + var victim = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow); + interposer.OnSecondCreate(() => + { + using var db = sql.NewContext(); + db.Database.ExecuteSqlRaw( + "UPDATE Deployment SET Status = {0}, FailureReason = N'closed as lost' WHERE DeploymentId = {1}", + (int)DeploymentStatus.TimedOut, victim.Value); + }); + coordinator.Tell(new DispatchDeployment(victim, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => interposer.Fired.ShouldBeTrue(), duration: PresenceBudget); + + // Positive control + ordering: a live row dispatched after it IS flipped, so the victim's dispatch was processed. + var live = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow); + coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = sql.NewContext(); + db.Deployments.AsNoTracking().Single(d => d.DeploymentId == live.Value).Status + .ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + using var verify = sql.NewContext(); + verify.Deployments.AsNoTracking().Single(d => d.DeploymentId == victim.Value).Status + .ShouldBe(DeploymentStatus.TimedOut, "the closure must not be reopened"); + verify.NodeDeploymentStates.Count(s => s.DeploymentId == victim.Value).ShouldBe(0); + } + + /// Runs an action just before handing out the SECOND context after arming (HandleDispatch: guard, then flip). + private sealed class InterposingDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private Action? _action; + private int _countdown; + public volatile bool Fired; + + public void OnSecondCreate(Action action) + { + _action = action; + Volatile.Write(ref _countdown, 2); + } + + public OtOpcUaConfigDbContext CreateDbContext() + { + if (Volatile.Read(ref _countdown) > 0 && Interlocked.Decrement(ref _countdown) == 0) + { + _action?.Invoke(); + Fired = true; + } + return inner.CreateDbContext(); + } + } + private static DeploymentId SeedDeployment( IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) {