diff --git a/Directory.Packages.props b/Directory.Packages.props index 5b95a7e1..d63e819e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -35,6 +35,12 @@ + + @@ -82,6 +88,8 @@ + + diff --git a/docs/ScriptEditor.md b/docs/ScriptEditor.md index 5321aca8..eb6985e2 100644 --- a/docs/ScriptEditor.md +++ b/docs/ScriptEditor.md @@ -162,11 +162,54 @@ user's fault. `ForbiddenTypeAnalyzer.Analyze(compilation)` walks the wrapped syntax tree and resolves every referenced symbol against the sandbox deny-list (namespace prefixes `System.IO`, `System.Net`, `System.Diagnostics`, `System.Reflection`, -`System.Threading.Tasks`, etc., plus type-granular denials like -`System.Environment` and `System.Threading.Thread`). Each violation is mapped +`System.Threading` — the whole namespace, with only `CancellationToken`, +`Interlocked` and `Volatile` allowed back in — `System.Threading.Tasks`, +`System.Runtime.InteropServices`, `System.Runtime.Loader`, `Microsoft.Win32`; +type-granular denials `System.Environment`, `System.AppDomain`, `System.GC`, +`System.Activator`, `System.Runtime.CompilerServices.Unsafe` / +`RuntimeHelpers`; and member-granular denials on `System.Type`'s by-name +reflection entry points — `GetType(string)`, `GetMethod(s)`, `InvokeMember`, +and the `GetProperty`/`GetField`/`GetMember`/`GetConstructor`/`GetEvent` +families). The runtime publish gate uses the same analyzer, so a deny-list +change needs a `DiagnoseTests` case to keep editor ⇔ publish parity. Each violation is mapped via `tree.GetMappedLineSpan(span)` to user-source coordinates and emitted as an error marker with code `OTSCRIPT_FORBIDDEN`. +### Runtime safety (what happens after publish) + +The editor cannot catch a script that compiles cleanly but never returns or +allocates without bound, so every runtime seam (virtual tags, Calculation tags, +scripted-alarm predicates) runs scripts through the shared `ScriptRunGuard`: + +- **Timeout + one orphan.** A run that exceeds its budget is abandoned (it cannot + be pre-empted in-process) and the tag/alarm reads Bad. While that abandoned run + is still executing, every further trigger is **refused** — Bad + `"previous evaluation still running"`, counted on `otopcua.script.run_refused` — + so a `while(true)` costs one pool thread, not one per trigger. +- **Quarantine after 3 consecutive timeouts** (`Scripting:QuarantineAfterConsecutiveTimeouts`). + Only a timeout is a strike: a refusal neither adds to nor resets the streak, and + a run that returns (value or throw) resets it. A quarantined script is never + dispatched again; it reads Bad `"script quarantined: …"`, logs one Error and + increments `otopcua_script_quarantined`. +- **Stuck-orphan quarantine.** A never-returning script (`while(true)`) times out + once and is then only refused, so the timeout streak alone never reaches 3. + Instead, a refusal behind an orphan that has been running longer than + `Scripting:StuckOrphanQuarantineFactor` × the timeout (default 10 × — 20 s at the + 2 s default) quarantines immediately, through the same path (Error log, metric + `reason=stuck`, Bad). A short overrun (a young orphan) stays a plain refusal and + the script resumes once the orphan completes. +- **Allocation budget.** A single run allocating more than + `Scripting:MaxAllocBytesPerRun` (64 MB) is quarantined. This is sampled after the + run returns, so a run that OOMs the process first is not stopped — that needs an + out-of-process runner. +- **Lifting.** A quarantine lifts when the script source changes or on redeploy, + never by itself. +- **Script-log rate limit.** The cluster-wide script-log fan-out admits 20 entries + per source per minute; the rest are counted and replaced by one "N suppressed" + summary. The summary is emitted on that source's **next** entry after the window + rolls, so a source that bursts and then goes quiet shows its summary only when + it logs again (the full stream is always in `scripts-*.log`). + ### 3. DependencyExtractor `DependencyExtractor.Extract(code).Rejections` enumerates every `ctx.GetTag` / diff --git a/docs/VirtualTags.md b/docs/VirtualTags.md index d58013ea..183fcef0 100644 --- a/docs/VirtualTags.md +++ b/docs/VirtualTags.md @@ -28,15 +28,15 @@ User scripts are compiled via `Microsoft.CodeAnalysis.CSharp` (regular compiler, `ScriptSandbox.Build` constructs the compile reference set in two parts. First, four pinned OtOpcUa assemblies are always included: `Core.Abstractions` (for `DataValueSnapshot` / `DriverDataType`), `Core.Scripting` (for `ScriptContext` + `Deadband`), `Serilog` (for `ILogger`), and the concrete context type's assembly. Second, the BCL subset is enumerated from the runtime's `TRUSTED_PLATFORM_ASSEMBLIES` list, restricted to filenames starting with `System.*` plus `netstandard.dll`, `mscorlib.dll`, and `Microsoft.Win32.Registry.dll` (the last needed so `ForbiddenTypeAnalyzer` can resolve and reject registry types). Pre-imported namespaces: `System`, `System.Linq`, `ZB.MOM.WW.OtOpcUa.Core.Abstractions`, `ZB.MOM.WW.OtOpcUa.Core.Scripting`. -`ForbiddenTypeAnalyzer.ForbiddenNamespacePrefixes` denies `System.IO`, `System.Net`, `System.Diagnostics`, `System.Reflection`, `System.Threading.Tasks`, `System.Runtime.InteropServices`, `System.Runtime.Loader`, and `Microsoft.Win32`. Matching is by prefix against the resolved symbol's containing namespace, so `System.Net` catches `System.Net.Http.HttpClient` and every subnamespace. `System.Threading.Tasks` is denied because scripts are synchronous predicates with no legitimate need to start background tasks — a `Task.Run` fan-out would outlive the per-evaluation timeout entirely (Core.Scripting-003). `System.Runtime.Loader` is denied to block `AssemblyLoadContext` / `AssemblyDependencyResolver` — arbitrary DLL loads into the host process (Core.Scripting-012). +`ForbiddenTypeAnalyzer.ForbiddenNamespacePrefixes` denies `System.IO`, `System.Net`, `System.Diagnostics`, `System.Reflection`, `System.Threading` (the whole namespace — `ForbiddenTypeAnalyzer.AllowedTypesInForbiddenNamespaces` lets only `CancellationToken`, `Interlocked` and `Volatile` back in), `System.Threading.Tasks`, `System.Runtime.InteropServices`, `System.Runtime.Loader`, and `Microsoft.Win32`. Matching is by prefix against the resolved symbol's containing namespace, so `System.Net` catches `System.Net.Http.HttpClient` and every subnamespace. `System.Threading.Tasks` is denied because scripts are synchronous predicates with no legitimate need to start background tasks — a `Task.Run` fan-out would outlive the per-evaluation timeout entirely (Core.Scripting-003). `System.Runtime.Loader` is denied to block `AssemblyLoadContext` / `AssemblyDependencyResolver` — arbitrary DLL loads into the host process (Core.Scripting-012). -`ForbiddenTypeAnalyzer.ForbiddenFullTypeNames` denies type-granularly: `System.Environment`, `System.AppDomain`, `System.GC`, `System.Activator`, `System.Threading.Thread`, `System.Threading.ThreadPool`, and `System.Threading.Timer`. These types require granular denial rather than namespace-prefix denial for different reasons: `Environment` / `AppDomain` / `GC` / `Activator` live directly in the `System` namespace (which is otherwise allowed for primitives), so a namespace-prefix rule cannot reach them without blocking `int` / `string` / `Math`; `Thread` / `ThreadPool` / `Timer` live in `System.Threading` (shared with allowed types like `CancellationToken` and `SemaphoreSlim`), so a prefix on `System.Threading` would block those too. `Environment.Exit` / `FailFast` terminate the host process outright (Core.Scripting-001); `Thread` and `ThreadPool` reintroduce background-fanout vectors that `System.Threading.Tasks` denial closed (Core.Scripting-010 / -012). +`ForbiddenTypeAnalyzer.ForbiddenFullTypeNames` denies type-granularly: `System.Environment`, `System.AppDomain`, `System.GC`, `System.Activator`, `System.Runtime.CompilerServices.Unsafe`, and `System.Runtime.CompilerServices.RuntimeHelpers`. These live in namespaces that are otherwise allowed — `System` (primitives, `Math`) and `System.Runtime.CompilerServices` (compile-time attributes such as `CallerMemberNameAttribute`) — so a namespace-prefix rule cannot reach them. `Environment.Exit` / `FailFast` terminate the host process outright (Core.Scripting-001); `Unsafe.As` bypasses the type system; `RuntimeHelpers.GetUninitializedObject` builds an instance of any type without a constructor. `Thread` / `ThreadPool` / `Timer` / `SemaphoreSlim` and the rest of `System.Threading` are now caught by the namespace prefix above instead of being listed one by one — a script is a synchronous expression with no use for background work or blocking primitives (Core.Scripting-010 / -012, review R05:10). `ForbiddenTypeAnalyzer.ForbiddenMembers` additionally denies `System.Type`'s by-name reflection entry points (`GetType(string)`, `GetMethod(s)`, `InvokeMember`, and the `GetProperty`/`GetField`/`GetMember`/`GetConstructor`/`GetEvent` families), which would otherwise reach a forbidden type without naming it syntactically. #### Known resource limits (accepted trade-offs) -The sandbox cannot prevent a script from **allocating unbounded memory**. A script calling `new byte[int.MaxValue]` repeatedly, or accumulating a large LINQ enumeration, can drive the server process to `OutOfMemoryException` before the 250 ms timeout fires. Script authoring is gated behind the Admin permission as the primary control; the test-harness preview (Stream F.4) allows operators to exercise a script before publishing. Out-of-process script execution is a v3 concern. +The sandbox cannot pre-empt a script. **Memory:** each run's own-thread allocation is sampled (`GC.GetAllocatedBytesForCurrentThread`) and a run over `Scripting:MaxAllocBytesPerRun` (default 64 MB) quarantines the script — but the sample is taken after the run returns, so a script that allocates fast enough (`new byte[int.MaxValue]` in a loop) can still OOM the process first. **CPU:** a tight loop keeps its pool thread after the timeout fires; `ScriptRunGuard` bounds that to one orphan per script by refusing further runs while it spins (Bad `"previous evaluation still running"`) and quarantines after three consecutive timeouts (a refusal is not a strike) — or immediately on a refusal behind an orphan that has run longer than `Scripting:StuckOrphanQuarantineFactor` (default 10) × the timeout, which is what quarantines a never-returning `while(true)`. Both are mitigations; pre-emptive CPU/memory limits need an out-of-process runner. Script authoring is gated behind the Admin permission as the primary control. See `docs/ScriptEditor.md` §"Runtime safety". -Similarly, **`System.Threading.Tasks` is now denied** (Core.Scripting-003), which prevents `Task.Run` / `Parallel` fan-out that would spawn background work outliving the timeout. However, a tight CPU-bound loop still runs on its thread-pool thread after `WaitAsync` returns — see the `TimedScriptEvaluator` remarks for detail. The orphaned thread is reclaimed when the Roslyn runtime eventually returns; in practice the operator fixes the script once the structured timeout warning appears in `scripts-*.log`. +`System.Threading.Tasks` is denied (Core.Scripting-003), which prevents `Task.Run` / `Parallel` fan-out that would spawn background work outliving the timeout. ### Compile cache (`CompiledScriptCache`) diff --git a/docs/security.md b/docs/security.md index 87ac8d22..fa8269a2 100644 --- a/docs/security.md +++ b/docs/security.md @@ -6,11 +6,14 @@ > auth + role-grant pages live in `OtOpcUa.AdminUI`), and Admin auth policies > register from `OtOpcUa.Host/Program.cs` via `AddOtOpcUaAuth` > (`src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs`) rather -> than in a separate Admin process. The Admin UI uses a **single Cookie -> authentication scheme** — there is no `AddJwtBearer` pipeline. The -> `Security:Jwt` section configures `JwtTokenService`, which mints a JWT at the -> `/auth/token` endpoint for **external** consumers (OPC UA clients / automation -> scripts); the cookie itself stores the `ClaimsPrincipal` directly. DataProtection +> than in a separate Admin process. The Admin UI session is a **Cookie** +> scheme; the cookie stores the `ClaimsPrincipal` directly. JWT is **opt-in by +> `Security:Jwt:SigningKey`** (validated at host start, ≥32 bytes): with a key, +> `/auth/token` mints an HS256 token for a **cookie session only** (a bearer +> cannot renew itself), and a first-party `AddJwtBearer` scheme (`Bearer`, +> `ValidAlgorithms = HS256`, parameters from `JwtTokenService.BuildValidationParameters`) +> accepts it on `/api/*` and every `RequireAuthorization` endpoint through the +> `OtOpcUa.CookieOrBearer` default policy scheme. Without a key neither exists. DataProtection > keys persist to the shared Config DB (`PersistKeysToDbContext`) > so cookies survive failover between admin-role nodes. > @@ -245,6 +248,25 @@ with two `Security:ApiKeys` knobs: a completed lookup (a hit or an authoritative miss) is stored, and a cached hit whose `ExpiresUtc` has since passed is re-read rather than served, so the TTL can never resurrect an expired key. +#### Pre-authentication guards (`KnownIdSet` / `MissBurst` / `MissRefillPerSecond`) + +The negative cache above is keyed by **key id**, so a flood of *distinct*, well-formed `otua__x` +credentials missed it every time and cost one indexed SQL query per attempt — under the SDK's +`SessionManager`-wide lock, with no credential required (review 03 §3.3). `BoundedApiKeyStore` now puts two +guards in front of the store, bound from the same `Security:ApiKeys` section +(`ApiKeyStoreGuardOptions`, Host-owned): + +| Key | Default | Meaning | +|---|---|---| +| `KnownIdSet` | `true` | Keep an in-memory set of every `ApiKeyId` the ConfigDb holds, refreshed every `StoreCacheSeconds` (single-flight; a failed refresh keeps the previous set). An id outside it is refused in memory — `otopcua.opcua.apikey_store.bounded{outcome=unknown_id}` — and the store is never asked. A key minted less than `StoreCacheSeconds` ago is refused until the next refresh (the same window `Enabled` revocation already has). `StoreCacheSeconds = 0` disables the set with the cache. | +| `MissBurst` | `100` | Token bucket on lookups that **reach** the store and **miss**, for when the set is unavailable (not loaded yet, or the store cannot list ids). Exhausted ⇒ unknown ids are refused without a store read (`{outcome=miss_budget}`, a Warning-free deny through the authenticator's catch-all). A hit refunds its token, so legitimate clients are never charged. `0` disables the budget. | +| `MissRefillPerSecond` | `20` | Refill rate of the miss bucket. | + +The budget is **process-wide, not per remote endpoint**: the OPC UA SDK's impersonation callback (the +`IOpcUaUserAuthenticator` seam) does not carry the client's address, so per-endpoint accounting needs that +seam widened in the OpcUaServer project (a Wave 3 follow-up). Neither guard applies to a driver-only node: +its `ArtifactApiKeyStore` is two in-memory dictionary probes and is never wrapped. + ### LDAP bind flow (`OtOpcUaLdapAuthService`) LDAP is configured under the `Security:Ldap` section (bound to `LdapOptions`, `src/Server/ZB.MOM.WW.OtOpcUa.Security/Ldap/LdapOptions.cs`, `SectionName = "Security:Ldap"`). The app authenticator is `OtOpcUaLdapAuthService` — a thin wrapper around the shared `ZB.MOM.WW.Auth.Ldap` directory client that adds two app-only concerns the shared library deliberately does not model: the `Enabled` master switch and `DevStubMode`. Since #520 it serves the **Admin UI cookie login (`/auth/login`) only** — `LdapOpcUaUserAuthenticator` is deleted and the OPC UA UserName path authenticates API keys instead (see [API-key identity](#api-key-identity-opc-ua-data-plane)). @@ -1116,6 +1138,31 @@ See `archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md` (R3-01) for the fu record, and `docs/Configuration.md` §`MeshTransport` / §`ConfigSource`-`ConfigServe` and `docs/Telemetry.md` for the affected listener ports. +### The known-cluster gate is fail-OPEN until central's first ClusterNode load (review 03 §4.2) + +`CentralCommandAuthInterceptor` verifies an inbound deploy ack against the PSK of the cluster named in +`x-otopcua-cluster`. `MeshPskProvider` consults a **known-cluster gate** first — the set of cluster ids +central read from `ClusterNode` on its last contact refresh — and refuses anything outside it in O(1), +touching neither the config map nor the secret store. That gate is **inactive until the first successful +load** (`CentralCommunicationActor` publishes it from `ContactsLoaded`; an empty publication also +deactivates it, deliberately, so "the DB told me nothing" never becomes a fleet-wide authentication +outage). During a central boot with the ConfigDb down, therefore: + +- an unauthenticated caller can name *any* `x-otopcua-cluster` value and make central perform one bounded + (3 s) secret-store read plus one Warning line per request — the cost/oracle the gate exists to remove; +- **keys are still required**: nothing is admitted without the matching PSK, so this is a cost exposure, + not an authentication bypass; +- the window is bounded by the bootstrap retry (15 × 2 s) and the `ContactRefreshSeconds` (60 s) periodic + refresh, and closes on the first load that succeeds. The actor logs the condition on every failed load + (*"The known-cluster gate was NOT refreshed either … it is INACTIVE"*). + +Since review 03 §3.4 the per-cluster keys are **refreshed asynchronously** from the same handler +(`IMeshPskProvider.RefreshAsync`: concurrent bounded reads, atomic snapshot swap, a failed read keeps the +previous key) instead of being invalidated on every refresh, so the first ack per cluster per minute no +longer pays a synchronous store read on a gRPC thread. Accepted as documented, bounded behaviour; the +alternative — refusing every cluster until SQL answers — would turn a ConfigDb outage into a deploy-ack +outage on top of it. + --- ## OTOPCUA0001 Analyzer — Compile-Time Guard diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs index 19d83728..d3044244 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs @@ -26,7 +26,11 @@ public sealed class Deployment public DateTime CreatedAtUtc { get; init; } = DateTime.UtcNow; /// Gets or sets the serialized artifact blob containing the configuration. - public byte[] ArtifactBlob { get; init; } = Array.Empty(); + /// + /// Settable (not init-only) so the retention sweep can EMPTY it on an old terminal row. An empty blob + /// is "no artifact" to every reader (#485), so the audit row survives with its bytes retired. + /// + public byte[] ArtifactBlob { get; set; } = Array.Empty(); /// Gets or sets the row version for optimistic concurrency control. public byte[] RowVersion { get; set; } = Array.Empty(); diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs new file mode 100644 index 00000000..db54f3a9 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs @@ -0,0 +1,1618 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ZB.MOM.WW.OtOpcUa.Configuration; + +#nullable disable + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations +{ + [DbContext(typeof(OtOpcUaConfigDbContext))] + [Migration("20261022102459_EnableSnapshotIsolation")] + partial class EnableSnapshotIsolation + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("Microsoft.AspNetCore.DataProtection.EntityFrameworkCore.DataProtectionKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("FriendlyName") + .HasColumnType("nvarchar(max)"); + + b.Property("Xml") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("DataProtectionKeys", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ApiKey", b => + { + b.Property("ApiKeyRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ApiKeyId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedUtc") + .HasColumnType("datetime2"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("ExpiresUtc") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SecretHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.HasKey("ApiKeyRowId"); + + b.HasIndex("ApiKeyId") + .IsUnique() + .HasDatabaseName("UX_ApiKey_ApiKeyId"); + + b.HasIndex("Principal") + .HasDatabaseName("IX_ApiKey_Principal"); + + b.ToTable("ApiKey", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("AkkaPort") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValue(4053); + + b.Property("ApplicationUri") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("DashboardPort") + .HasColumnType("int"); + + b.Property("DriverConfigOverridesJson") + .HasColumnType("nvarchar(max)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("GrpcPort") + .HasColumnType("int"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("LastSeenAt") + .HasColumnType("datetime2(3)"); + + b.Property("MaintenanceMode") + .HasColumnType("bit"); + + b.Property("OpcUaPort") + .HasColumnType("int"); + + b.Property("ServiceLevelBase") + .HasColumnType("tinyint"); + + b.HasKey("NodeId"); + + b.HasIndex("ApplicationUri") + .IsUnique() + .HasDatabaseName("UX_ClusterNode_ApplicationUri"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_ClusterNode_ClusterId"); + + b.ToTable("ClusterNode", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNodeCredential", b => + { + b.Property("CredentialId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("NodeId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RotatedAt") + .HasColumnType("datetime2(3)"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.HasKey("CredentialId"); + + b.HasIndex("Kind", "Value") + .IsUnique() + .HasDatabaseName("UX_ClusterNodeCredential_Value") + .HasFilter("[Enabled] = 1"); + + b.HasIndex("NodeId", "Enabled") + .HasDatabaseName("IX_ClusterNodeCredential_NodeId"); + + b.ToTable("ClusterNodeCredential", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ConfigAuditLog", b => + { + b.Property("AuditId") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("AuditId")); + + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CorrelationId") + .HasColumnType("uniqueidentifier"); + + b.Property("DetailsJson") + .HasColumnType("nvarchar(max)"); + + b.Property("EventId") + .HasColumnType("uniqueidentifier"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("GenerationId") + .HasColumnType("bigint"); + + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Outcome") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Timestamp") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.HasKey("AuditId"); + + b.HasIndex("EventId") + .IsUnique() + .HasDatabaseName("UX_ConfigAuditLog_EventId") + .HasFilter("[EventId] IS NOT NULL"); + + b.HasIndex("GenerationId") + .HasDatabaseName("IX_ConfigAuditLog_Generation") + .HasFilter("[GenerationId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Timestamp") + .IsDescending(false, true) + .HasDatabaseName("IX_ConfigAuditLog_Cluster_Time"); + + b.ToTable("ConfigAuditLog", null, t => + { + t.HasCheckConstraint("CK_ConfigAuditLog_DetailsJson_IsJson", "DetailsJson IS NULL OR ISJSON(DetailsJson) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ConfigEdit", b => + { + b.Property("EditId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("EditedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("EditedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EntityId") + .HasColumnType("uniqueidentifier"); + + b.Property("EntityType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ExecutionId") + .HasColumnType("uniqueidentifier"); + + b.Property("FieldsJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SourceNode") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("EditId"); + + b.HasIndex("EditedAtUtc") + .HasDatabaseName("IX_ConfigEdit_EditedAt"); + + b.HasIndex("ExecutionId") + .HasDatabaseName("IX_ConfigEdit_Execution") + .HasFilter("[ExecutionId] IS NOT NULL"); + + b.HasIndex("EntityType", "EntityId") + .HasDatabaseName("IX_ConfigEdit_Entity"); + + b.ToTable("ConfigEdit", null, t => + { + t.HasCheckConstraint("CK_ConfigEdit_FieldsJson_IsJson", "ISJSON(FieldsJson) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Deployment", b => + { + b.Property("DeploymentId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ArtifactBlob") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("CreatedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("FailureReason") + .HasMaxLength(2048) + .HasColumnType("nvarchar(2048)"); + + b.Property("RevisionHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SealedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("DeploymentId"); + + b.HasIndex("CreatedAtUtc") + .HasDatabaseName("IX_Deployment_CreatedAt"); + + b.HasIndex("Status") + .HasDatabaseName("IX_Deployment_Status"); + + b.ToTable("Deployment", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Device", b => + { + b.Property("DeviceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DeviceConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DeviceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverInstanceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("DeviceRowId"); + + b.HasIndex("DeviceId") + .IsUnique() + .HasDatabaseName("UX_Device_LogicalId") + .HasFilter("[DeviceId] IS NOT NULL"); + + b.HasIndex("DriverInstanceId") + .HasDatabaseName("IX_Device_Driver"); + + b.HasIndex("DriverInstanceId", "Name") + .IsUnique() + .HasDatabaseName("UX_Device_Driver_Name"); + + b.ToTable("Device", null, t => + { + t.HasCheckConstraint("CK_Device_DeviceConfig_IsJson", "ISJSON(DeviceConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverInstance", b => + { + b.Property("DriverInstanceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DriverInstanceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ResilienceConfig") + .HasColumnType("nvarchar(max)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("DriverInstanceRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_DriverInstance_Cluster"); + + b.HasIndex("DriverInstanceId") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_LogicalId") + .HasFilter("[DriverInstanceId] IS NOT NULL"); + + b.HasIndex("RawFolderId") + .HasDatabaseName("IX_DriverInstance_RawFolder"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_ClusterRoot_Name") + .HasFilter("[RawFolderId] IS NULL"); + + b.HasIndex("ClusterId", "RawFolderId", "Name") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_Folder_Name") + .HasFilter("[RawFolderId] IS NOT NULL"); + + b.ToTable("DriverInstance", null, t => + { + t.HasCheckConstraint("CK_DriverInstance_DriverConfig_IsJson", "ISJSON(DriverConfig) = 1"); + + t.HasCheckConstraint("CK_DriverInstance_ResilienceConfig_IsJson", "ResilienceConfig IS NULL OR ISJSON(ResilienceConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Equipment", b => + { + b.Property("EquipmentRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AssetLocation") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("DeviceManualUri") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentClassRef") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EquipmentId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("EquipmentUuid") + .HasColumnType("uniqueidentifier"); + + b.Property("HardwareRevision") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("MachineCode") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Manufacturer") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ManufacturerUri") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Model") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SAPID") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SerialNumber") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SoftwareRevision") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("UnsLineId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("YearOfConstruction") + .HasColumnType("smallint"); + + b.Property("ZTag") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("EquipmentRowId"); + + b.HasIndex("EquipmentId") + .IsUnique() + .HasDatabaseName("UX_Equipment_LogicalId") + .HasFilter("[EquipmentId] IS NOT NULL"); + + b.HasIndex("EquipmentUuid") + .IsUnique() + .HasDatabaseName("UX_Equipment_Uuid"); + + b.HasIndex("MachineCode") + .HasDatabaseName("IX_Equipment_MachineCode"); + + b.HasIndex("SAPID") + .HasDatabaseName("IX_Equipment_SAPID") + .HasFilter("[SAPID] IS NOT NULL"); + + b.HasIndex("UnsLineId") + .HasDatabaseName("IX_Equipment_Line"); + + b.HasIndex("ZTag") + .HasDatabaseName("IX_Equipment_ZTag") + .HasFilter("[ZTag] IS NOT NULL"); + + b.HasIndex("UnsLineId", "Name") + .IsUnique() + .HasDatabaseName("UX_Equipment_LinePath"); + + b.ToTable("Equipment", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ExternalIdReservation", b => + { + b.Property("ReservationId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("EquipmentUuid") + .HasColumnType("uniqueidentifier"); + + b.Property("FirstPublishedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("FirstPublishedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("LastPublishedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("ReleaseReason") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("ReleasedAt") + .HasColumnType("datetime2(3)"); + + b.Property("ReleasedBy") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("ReservationId"); + + b.HasIndex("EquipmentUuid") + .HasDatabaseName("IX_ExternalIdReservation_Equipment"); + + b.HasIndex("Kind", "Value") + .IsUnique() + .HasDatabaseName("UX_ExternalIdReservation_KindValue_Active") + .HasFilter("[ReleasedAt] IS NULL"); + + b.ToTable("ExternalIdReservation", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.LdapGroupRoleMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("IsSystemWide") + .HasColumnType("bit"); + + b.Property("LdapGroup") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Role") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("Id"); + + b.HasIndex("ClusterId"); + + b.HasIndex("LdapGroup") + .HasDatabaseName("IX_LdapGroupRoleMapping_Group"); + + b.HasIndex("LdapGroup", "ClusterId") + .IsUnique() + .HasDatabaseName("UX_LdapGroupRoleMapping_Group_Cluster") + .HasFilter("[ClusterId] IS NOT NULL"); + + b.ToTable("LdapGroupRoleMapping", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeAcl", b => + { + b.Property("NodeAclRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("NodeAclId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("PermissionFlags") + .HasColumnType("int"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScopeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ScopeKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.HasKey("NodeAclRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_NodeAcl_Cluster"); + + b.HasIndex("NodeAclId") + .IsUnique() + .HasDatabaseName("UX_NodeAcl_LogicalId") + .HasFilter("[NodeAclId] IS NOT NULL"); + + b.HasIndex("Principal") + .HasDatabaseName("IX_NodeAcl_Principal"); + + b.HasIndex("ScopeKind", "ScopeId") + .HasDatabaseName("IX_NodeAcl_Scope") + .HasFilter("[ScopeId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Principal", "ScopeKind", "ScopeId") + .IsUnique() + .HasDatabaseName("UX_NodeAcl_PrincipalScope") + .HasFilter("[ScopeId] IS NOT NULL"); + + b.ToTable("NodeAcl", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeDeploymentState", b => + { + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DeploymentId") + .HasColumnType("uniqueidentifier"); + + b.Property("AppliedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("FailureReason") + .HasMaxLength(2048) + .HasColumnType("nvarchar(2048)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("StartedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("NodeId", "DeploymentId"); + + b.HasIndex("DeploymentId") + .HasDatabaseName("IX_NodeDeploymentState_Deployment"); + + b.HasIndex("Status") + .HasDatabaseName("IX_NodeDeploymentState_Status"); + + b.ToTable("NodeDeploymentState", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.PollGroup", b => + { + b.Property("PollGroupRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DriverInstanceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("IntervalMs") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PollGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("PollGroupRowId"); + + b.HasIndex("DriverInstanceId") + .HasDatabaseName("IX_PollGroup_Driver"); + + b.HasIndex("PollGroupId") + .IsUnique() + .HasDatabaseName("UX_PollGroup_LogicalId") + .HasFilter("[PollGroupId] IS NOT NULL"); + + b.ToTable("PollGroup", null, t => + { + t.HasCheckConstraint("CK_PollGroup_IntervalMs_Min", "IntervalMs >= 50"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.RawFolder", b => + { + b.Property("RawFolderRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ParentRawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.HasKey("RawFolderRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_RawFolder_Cluster"); + + b.HasIndex("RawFolderId") + .IsUnique() + .HasDatabaseName("UX_RawFolder_LogicalId") + .HasFilter("[RawFolderId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_RawFolder_ClusterRoot_Name") + .HasFilter("[ParentRawFolderId] IS NULL"); + + b.HasIndex("ClusterId", "ParentRawFolderId", "Name") + .IsUnique() + .HasDatabaseName("UX_RawFolder_Parent_Name") + .HasFilter("[ParentRawFolderId] IS NOT NULL"); + + b.ToTable("RawFolder", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Script", b => + { + b.Property("ScriptRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("Language") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SourceCode") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SourceHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("ScriptRowId"); + + b.HasIndex("ScriptId") + .IsUnique() + .HasDatabaseName("UX_Script_LogicalId") + .HasFilter("[ScriptId] IS NOT NULL"); + + b.HasIndex("SourceHash") + .HasDatabaseName("IX_Script_SourceHash"); + + b.ToTable("Script", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ScriptedAlarm", b => + { + b.Property("ScriptedAlarmRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AlarmType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("HistorizeToAveva") + .HasColumnType("bit"); + + b.Property("MessageTemplate") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PredicateScriptId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Retain") + .HasColumnType("bit"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptedAlarmId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Severity") + .HasColumnType("int"); + + b.HasKey("ScriptedAlarmRowId"); + + b.HasIndex("PredicateScriptId") + .HasDatabaseName("IX_ScriptedAlarm_Script"); + + b.HasIndex("ScriptedAlarmId") + .IsUnique() + .HasDatabaseName("UX_ScriptedAlarm_LogicalId") + .HasFilter("[ScriptedAlarmId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "Name") + .IsUnique() + .HasDatabaseName("UX_ScriptedAlarm_EquipmentPath"); + + b.ToTable("ScriptedAlarm", null, t => + { + t.HasCheckConstraint("CK_ScriptedAlarm_AlarmType", "AlarmType IN ('AlarmCondition','LimitAlarm','OffNormalAlarm','DiscreteAlarm')"); + + t.HasCheckConstraint("CK_ScriptedAlarm_Severity_Range", "Severity BETWEEN 1 AND 1000"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", b => + { + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Enterprise") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2(3)"); + + b.Property("ModifiedBy") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("NodeCount") + .HasColumnType("tinyint"); + + b.Property("Notes") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("RedundancyMode") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Site") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("ClusterId"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("UX_ServerCluster_Name"); + + b.HasIndex("Site") + .HasDatabaseName("IX_ServerCluster_Site"); + + b.ToTable("ServerCluster", null, t => + { + t.HasCheckConstraint("CK_ServerCluster_RedundancyMode_NodeCount", "((NodeCount = 1 AND RedundancyMode = 'None') OR (NodeCount = 2 AND RedundancyMode IN ('Warm', 'Hot')))"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Tag", b => + { + b.Property("TagRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AccessLevel") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("DataType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("DeviceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PollGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("TagConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("TagId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("WriteIdempotent") + .HasColumnType("bit"); + + b.HasKey("TagRowId"); + + b.HasIndex("TagId") + .IsUnique() + .HasDatabaseName("UX_Tag_LogicalId") + .HasFilter("[TagId] IS NOT NULL"); + + b.HasIndex("DeviceId", "Name") + .IsUnique() + .HasDatabaseName("UX_Tag_Device_Name") + .HasFilter("[TagGroupId] IS NULL"); + + b.HasIndex("DeviceId", "TagGroupId") + .HasDatabaseName("IX_Tag_Device"); + + b.HasIndex("DeviceId", "TagGroupId", "Name") + .IsUnique() + .HasDatabaseName("UX_Tag_Group_Name") + .HasFilter("[TagGroupId] IS NOT NULL"); + + b.ToTable("Tag", null, t => + { + t.HasCheckConstraint("CK_Tag_TagConfig_IsJson", "ISJSON(TagConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.TagGroup", b => + { + b.Property("TagGroupRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DeviceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ParentTagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.Property("TagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("TagGroupRowId"); + + b.HasIndex("DeviceId") + .HasDatabaseName("IX_TagGroup_Device"); + + b.HasIndex("TagGroupId") + .IsUnique() + .HasDatabaseName("UX_TagGroup_LogicalId") + .HasFilter("[TagGroupId] IS NOT NULL"); + + b.HasIndex("DeviceId", "Name") + .IsUnique() + .HasDatabaseName("UX_TagGroup_DeviceRoot_Name") + .HasFilter("[ParentTagGroupId] IS NULL"); + + b.HasIndex("DeviceId", "ParentTagGroupId", "Name") + .IsUnique() + .HasDatabaseName("UX_TagGroup_Parent_Name") + .HasFilter("[ParentTagGroupId] IS NOT NULL"); + + b.ToTable("TagGroup", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsArea", b => + { + b.Property("UnsAreaRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("UnsAreaId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsAreaRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_UnsArea_Cluster"); + + b.HasIndex("UnsAreaId") + .IsUnique() + .HasDatabaseName("UX_UnsArea_LogicalId") + .HasFilter("[UnsAreaId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_UnsArea_ClusterName"); + + b.ToTable("UnsArea", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsLine", b => + { + b.Property("UnsLineRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("UnsAreaId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("UnsLineId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsLineRowId"); + + b.HasIndex("UnsAreaId") + .HasDatabaseName("IX_UnsLine_Area"); + + b.HasIndex("UnsLineId") + .IsUnique() + .HasDatabaseName("UX_UnsLine_LogicalId") + .HasFilter("[UnsLineId] IS NOT NULL"); + + b.HasIndex("UnsAreaId", "Name") + .IsUnique() + .HasDatabaseName("UX_UnsLine_AreaName"); + + b.ToTable("UnsLine", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsTagReference", b => + { + b.Property("UnsTagReferenceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DisplayNameOverride") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.Property("TagId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("UnsTagReferenceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsTagReferenceRowId"); + + b.HasIndex("EquipmentId") + .HasDatabaseName("IX_UnsTagReference_Equipment"); + + b.HasIndex("TagId") + .HasDatabaseName("IX_UnsTagReference_Tag"); + + b.HasIndex("UnsTagReferenceId") + .IsUnique() + .HasDatabaseName("UX_UnsTagReference_LogicalId") + .HasFilter("[UnsTagReferenceId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "TagId") + .IsUnique() + .HasDatabaseName("UX_UnsTagReference_Equip_Tag"); + + b.ToTable("UnsTagReference", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.VirtualTag", b => + { + b.Property("VirtualTagRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ChangeTriggered") + .HasColumnType("bit"); + + b.Property("DataType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Historize") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("TimerIntervalMs") + .HasColumnType("int"); + + b.Property("VirtualTagId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("VirtualTagRowId"); + + b.HasIndex("ScriptId") + .HasDatabaseName("IX_VirtualTag_Script"); + + b.HasIndex("VirtualTagId") + .IsUnique() + .HasDatabaseName("UX_VirtualTag_LogicalId") + .HasFilter("[VirtualTagId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "Name") + .IsUnique() + .HasDatabaseName("UX_VirtualTag_EquipmentPath"); + + b.ToTable("VirtualTag", null, t => + { + t.HasCheckConstraint("CK_VirtualTag_TimerInterval_Min", "TimerIntervalMs IS NULL OR TimerIntervalMs >= 50"); + + t.HasCheckConstraint("CK_VirtualTag_Trigger_AtLeastOne", "ChangeTriggered = 1 OR TimerIntervalMs IS NOT NULL"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany("Nodes") + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNodeCredential", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", "Node") + .WithMany("Credentials") + .HasForeignKey("NodeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Node"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverInstance", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.LdapGroupRoleMapping", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeDeploymentState", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Deployment", "Deployment") + .WithMany() + .HasForeignKey("DeploymentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", "Node") + .WithMany() + .HasForeignKey("NodeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Deployment"); + + b.Navigation("Node"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.RawFolder", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", null) + .WithMany("RawFolders") + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsArea", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.Navigation("Credentials"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", b => + { + b.Navigation("Nodes"); + + b.Navigation("RawFolders"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs new file mode 100644 index 00000000..58cf9a7e --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs @@ -0,0 +1,36 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations +{ + /// + /// Enables ALLOW_SNAPSHOT_ISOLATION on the config database so the deploy path can read every + /// table inside one SNAPSHOT transaction (DraftSnapshotFactory.FromConfigDbAsync): + /// the validator and the artifact composer then see one consistent state instead of two reads + /// an AdminUI save can land between. No schema change — the model snapshot is untouched. + /// + /// + /// ALTER DATABASE cannot run inside the user transaction the migrator wraps each migration in, + /// hence suppressTransaction: true. Turning the option ON does not need exclusive access (unlike + /// READ_COMMITTED_SNAPSHOT); it waits for in-flight transactions and then takes effect. Down + /// leaves it ON: a database that has run with it is not harmed by keeping it, and switching it OFF + /// would fail any deploy running at that instant. + /// + public partial class EnableSnapshotIsolation : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + "ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON;", + suppressTransaction: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + // Deliberately a no-op; see the class remarks. + } + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs index 0c0c47d9..fd7dc35a 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs @@ -66,6 +66,23 @@ public sealed class DraftSnapshot /// silently dropped when the permission trie is built, so the grant has to be caught here instead. public IReadOnlyList NodeAcls { get; init; } = []; + /// + /// Cluster rows. Not read by the row-level rules; carried so the deploy path validates the cluster + /// topology () and composes the artifact from + /// the SAME read as every other table — a second read between validate and compose is exactly how + /// an edit ships unvalidated. + /// + public IReadOnlyList ServerClusters { get; init; } = []; + + /// Cluster-node rows; see for why they ride the snapshot. + public IReadOnlyList ClusterNodes { get; init; } = []; + + /// + /// Data-plane API keys (peppered HMAC only, never a secret). Not validated; carried so the artifact + /// composer ships the keys a driver-only node authenticates against from the one consistent read. + /// + public IReadOnlyList ApiKeys { get; init; } = []; + /// Prior Equipment rows (any generation, same cluster) for stability checks. public IReadOnlyList PriorEquipment { get; init; } = []; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs index 3bfd0a5c..30873aca 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs @@ -1,3 +1,4 @@ +using System.Data; using Microsoft.EntityFrameworkCore; namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation; @@ -21,6 +22,16 @@ namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation; /// / are left null so the path-length rule uses its /// conservative upper bound. /// +/// +/// One consistent read. On a relational provider the tables are read inside a +/// transaction, so a /uns or /raw save that +/// lands mid-read cannot leave half the tables on the new state and half on the old. The deploy path +/// validates AND composes its artifact from this one snapshot, which is what makes "the artifact the +/// validator never saw" unrepresentable. SQL Server requires ALLOW_SNAPSHOT_ISOLATION ON +/// (migration EnableSnapshotIsolation); without it the first query fails with error 3952 and +/// the deploy is rejected rather than silently read unisolated. A non-relational provider (the +/// InMemory test provider) and a context already inside a transaction read without one. +/// /// public static class DraftSnapshotFactory { @@ -29,6 +40,29 @@ public static class DraftSnapshotFactory /// Cancellation token. /// A snapshot populated from the live DB, ready for . public static async Task FromConfigDbAsync(OtOpcUaConfigDbContext db, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(db); + + var ownsTransaction = db.Database.IsRelational() && db.Database.CurrentTransaction is null; + var tx = ownsTransaction + ? await db.Database.BeginTransactionAsync(IsolationLevel.Snapshot, ct).ConfigureAwait(false) + : null; + await using (tx) + { + var snapshot = await ReadAsync(db, ct).ConfigureAwait(false); + // Read-only: committing releases the snapshot's version-store reservation; nothing was written. + if (tx is not null) await tx.CommitAsync(ct).ConfigureAwait(false); + return snapshot; + } + } + + // Every composed table is read ORDERED BY ITS NATURAL KEY, server-side. That order IS the deployment + // artifact's order (ConfigComposer.Compose serialises the snapshot as given): it is exactly the order the + // pre-R03:3.2 composer's own queries produced — the database collation's, case-insensitive on SQL Server — + // so the revision hash of an unchanged configuration is byte-identical across the refactor (fix round 1; + // pinned by ConfigComposerSqlServerHashStabilityTests). Re-sorting in memory with any other comparer would + // re-hash every mixed-case configuration and make drift detection report the whole fleet drifted. + private static async Task ReadAsync(OtOpcUaConfigDbContext db, CancellationToken ct) => new DraftSnapshot { // GenerationId=0 and null Enterprise/Site are intentional conservative fallbacks at the @@ -36,23 +70,26 @@ public static class DraftSnapshotFactory // EquipmentUuid checks run in separate validator passes — no rule here reads these fields. GenerationId = 0, // generation model dropped; placeholder (no rule reads it) ClusterId = string.Empty, // global snapshot; rules compare entity ClusterId fields, not this + ServerClusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), + ClusterNodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), // v3: Namespace entity retired — no namespace list to materialize. The Raw tree // (RawFolders/TagGroups) + UnsTagReferences feed the v3 rules (charset, tagname length, // effective-leaf uniqueness). - RawFolders = await db.RawFolders.AsNoTracking().ToListAsync(ct), - DriverInstances = await db.DriverInstances.AsNoTracking().ToListAsync(ct), - Devices = await db.Devices.AsNoTracking().ToListAsync(ct), - TagGroups = await db.TagGroups.AsNoTracking().ToListAsync(ct), - UnsAreas = await db.UnsAreas.AsNoTracking().ToListAsync(ct), - UnsLines = await db.UnsLines.AsNoTracking().ToListAsync(ct), - Equipment = await db.Equipment.AsNoTracking().ToListAsync(ct), - Tags = await db.Tags.AsNoTracking().ToListAsync(ct), - UnsTagReferences = await db.UnsTagReferences.AsNoTracking().ToListAsync(ct), - VirtualTags = await db.VirtualTags.AsNoTracking().ToListAsync(ct), - ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().ToListAsync(ct), - Scripts = await db.Scripts.AsNoTracking().ToListAsync(ct), - PollGroups = await db.PollGroups.AsNoTracking().ToListAsync(ct), - NodeAcls = await db.NodeAcls.AsNoTracking().ToListAsync(ct), + RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), + DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), + Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), + TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), + UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), + UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), + Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), + Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), + UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), + VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), + ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), + PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), + NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), PriorEquipment = [], // intentional: no prior-generation table to diff against at this level ActiveReservations = await db.ExternalIdReservations .AsNoTracking() diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs index 04e08fe9..96a25da5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs @@ -452,6 +452,58 @@ public static class DraftValidator "Calculation tags form a dependency cycle (members: " + string.Join(", ", cycle) + "); a calc→calc cycle is a live oscillation loop and must be broken before deploy", cycle.Count > 0 ? cycle[0] : null)); + + // Rule 3 — effective-trigger gate (R07:U5). A calc tag evaluates on a declared dependency + // changing (change-trigger: the literal ctx.GetTag paths) and/or on a timer. A change-triggered + // tag whose script has NO literal reads (`return ctx.Now.Hour;`, a dynamic-path read) and no timer + // never evaluates at all: it sits BadWaitingForInitialData forever while the driver reports + // Healthy, and nothing warned. Same for a tag that is neither change- nor timer-triggered. + foreach (var (t, _, scriptId) in calcTags) + { + var (changeTriggered, timerIntervalMs) = ParseTriggers(t.TagConfig); + if (timerIntervalMs is > 0) continue; + var source = scriptId is not null && scriptSourceById.TryGetValue(scriptId, out var s) ? s : null; + // An unresolved scriptId is already CalculationScriptMissing / CalculationScriptNotFound; with no + // source there are no reads to count, so a change-triggered tag would get a second, misleading + // "no literal ctx.GetTag read" error. (changeTriggered=false + no timer is source-independent.) + if (changeTriggered && source is null) continue; + var depCount = EquipmentScriptPaths.ExtractDependencyRefs(source).Count; + if (changeTriggered && depCount > 0) continue; + errors.Add(new("CalculationNoEffectiveTrigger", + changeTriggered + ? $"Calculation tag '{t.TagId}' is change-triggered but its script has no literal ctx.GetTag read, " + + "and it has no timerIntervalMs — nothing would ever evaluate it. Add a dependency read or a timer." + : $"Calculation tag '{t.TagId}' has changeTriggered=false and no timerIntervalMs — nothing would ever evaluate it. " + + "Enable the change trigger or add a timer.", + t.TagId)); + } + } + + /// Read the trigger fields from a calc tag's TagConfig: changeTriggered + /// (default true, the driver's own default) and timerIntervalMs (null when absent or not a + /// number). Never throws — malformed JSON yields the defaults. + private static (bool ChangeTriggered, int? TimerIntervalMs) ParseTriggers(string? tagConfig) + { + if (string.IsNullOrWhiteSpace(tagConfig)) return (true, null); + try + { + using var doc = System.Text.Json.JsonDocument.Parse(tagConfig); + var root = doc.RootElement; + if (root.ValueKind != System.Text.Json.JsonValueKind.Object) return (true, null); + var changeTriggered = true; + if (root.TryGetProperty("changeTriggered", out var ct) + && ct.ValueKind is System.Text.Json.JsonValueKind.False or System.Text.Json.JsonValueKind.True) + changeTriggered = ct.GetBoolean(); + int? timer = null; + if (root.TryGetProperty("timerIntervalMs", out var tm) + && tm.ValueKind == System.Text.Json.JsonValueKind.Number && tm.TryGetInt32(out var ms)) + timer = ms; + return (changeTriggered, timer); + } + catch (System.Text.Json.JsonException) + { + return (true, null); + } } /// Extract the scriptId string from a calc tag's schemaless TagConfig JSON. Never diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs index 8a9b9ec4..6d239f75 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs @@ -92,6 +92,12 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl private bool? _lastGateDecision; private long _queuedRowCount; + // Cached dead-letter depth (R05:7). GetStatus used to run a synchronous COUNT(*) table scan per + // call — one per /healthz probe and per AdminUI poll on the shared LocalDb file. Kept in step by + // every path that flips dead_lettered (drain outcomes, corrupt rows, RetryDeadLettered, retention + // purge) and resynced from storage on the same cadence as _queuedRowCount, so a row the pair peer + // replicates in is visible after the next resync rather than never. + private long _deadLetterRowCount; // Probe counter — incremented every time we actually issue a real COUNT(*) for // capacity enforcement. Public for test instrumentation only. private long _capacityProbeCount; @@ -145,9 +151,16 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl _clock = clock ?? (() => DateTime.UtcNow); _drainGate = drainGate ?? (static () => true); - _queuedRowCount = ProbeQueuedRowCount(); + _queuedRowCount = ProbeRowCount(deadLettered: false); + _deadLetterRowCount = ProbeRowCount(deadLettered: true); } + /// Test-only: re-sync both cached counters (queued + dead-lettered) from storage — the same + /// resync the enqueue cadence drives every enqueues. + /// Cancellation token. + /// A task that completes once both counters match storage. + public Task DebugResyncCountersAsync(CancellationToken ct) => ResyncRowCountsAsync(ct); + /// /// Start the background drain worker. Not started automatically so tests can /// drive deterministically. @@ -260,7 +273,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl // has a second way to fall behind that no local code path can observe. if (enqueuesSinceResync >= ResyncEnqueueInterval) { - await ResyncQueuedRowCountAsync(ct).ConfigureAwait(false); + await ResyncRowCountsAsync(ct).ConfigureAwait(false); cached = Interlocked.Read(ref _queuedRowCount); Interlocked.Exchange(ref _enqueuesSinceResync, 0); } @@ -273,22 +286,26 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl await EnforceCapacityAsync(ct).ConfigureAwait(false); } - /// Synchronously query COUNT(*) of non-dead-lettered rows. Used at startup. - private long ProbeQueuedRowCount() + /// Synchronously query COUNT(*) of queued or dead-lettered rows. Used at startup. + private long ProbeRowCount(bool deadLettered) { Interlocked.Increment(ref _capacityProbeCount); using var conn = _db.CreateConnection(); using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0"; + cmd.CommandText = deadLettered + ? "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1" + : "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0"; return (long)(cmd.ExecuteScalar() ?? 0L); } - /// Re-sync the in-memory counter from storage (async path). - private async Task ResyncQueuedRowCountAsync(CancellationToken ct) + /// Re-sync both in-memory counters from storage (async path). + private async Task ResyncRowCountsAsync(CancellationToken ct) { Interlocked.Increment(ref _capacityProbeCount); var live = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false); Interlocked.Exchange(ref _queuedRowCount, live); + var dead = await CountAsync("dead_lettered = 1", ct).ConfigureAwait(false); + Interlocked.Exchange(ref _deadLetterRowCount, dead); } private async Task CountAsync(string predicate, CancellationToken ct) @@ -356,6 +373,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl await corruptTx.CommitAsync(ct).ConfigureAwait(false); } Interlocked.Add(ref _queuedRowCount, -corruptIds.Count); + Interlocked.Add(ref _deadLetterRowCount, corruptIds.Count); _logger.Warning( "Dead-lettered {Count} historian queue row(s) with un-deserializable payload", corruptIds.Count); @@ -405,6 +423,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl } int rowsLeavingQueue = 0; + int rowsDeadLettered = 0; await using (var tx = await _db.BeginTransactionAsync(ct).ConfigureAwait(false)) { for (var i = 0; i < outcomes.Count; i++) @@ -423,6 +442,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl case HistorianWriteOutcome.PermanentFail: await DeadLetterRowAsync(tx, id, $"permanent fail at {_clock():O}", ct).ConfigureAwait(false); rowsLeavingQueue++; + rowsDeadLettered++; break; case HistorianWriteOutcome.RetryPlease: // finding 002: cap retries so a perpetually-RetryPlease (poison) @@ -434,6 +454,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl { await DeadLetterRowAsync(tx, id, $"max attempts ({_maxAttempts}) exceeded", ct).ConfigureAwait(false); rowsLeavingQueue++; + rowsDeadLettered++; } else { @@ -449,6 +470,8 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl // cap (finding 002) — keep the counter aligned. if (rowsLeavingQueue > 0) Interlocked.Add(ref _queuedRowCount, -rowsLeavingQueue); + if (rowsDeadLettered > 0) + Interlocked.Add(ref _deadLetterRowCount, rowsDeadLettered); var acks = outcomes.Count(o => o == HistorianWriteOutcome.Ack); lock (_statusLock) @@ -523,13 +546,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl var queued = Interlocked.Read(ref _queuedRowCount); if (queued < 0) queued = 0; - long deadlettered; - using (var conn = _db.CreateConnection()) - using (var cmd = conn.CreateCommand()) - { - cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1"; - deadlettered = (long)(cmd.ExecuteScalar() ?? 0L); - } + // Cached, never a live scan (R05:7) — see the field's remarks for the resync cadence. + var deadlettered = Interlocked.Read(ref _deadLetterRowCount); + if (deadlettered < 0) deadlettered = 0; DateTime? lastDrain, lastSuccess; string? lastError; @@ -568,7 +587,11 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl var revived = cmd.ExecuteNonQuery(); // Dead-lettered rows rejoin the non-dead-lettered queue — keep the in-memory // counter aligned. - if (revived > 0) Interlocked.Add(ref _queuedRowCount, revived); + if (revived > 0) + { + Interlocked.Add(ref _queuedRowCount, revived); + Interlocked.Add(ref _deadLetterRowCount, -revived); + } return revived; } @@ -646,8 +669,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl Interlocked.Increment(ref _capacityProbeCount); var count = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false); - // Resync the in-memory counter while we have a fresh number. + // Resync the in-memory counters while we have a fresh number. Interlocked.Exchange(ref _queuedRowCount, count); + Interlocked.Exchange(ref _deadLetterRowCount, await CountAsync("dead_lettered = 1", ct).ConfigureAwait(false)); if (count < _capacity) return; var toEvict = count - _capacity + 1; @@ -684,7 +708,10 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl ct).ConfigureAwait(false); if (purged > 0) + { + Interlocked.Add(ref _deadLetterRowCount, -purged); _logger.Information("Purged {Count} dead-lettered row(s) past retention window", purged); + } } private void BumpBackoff() => _backoffIndex = Math.Min(_backoffIndex + 1, BackoffLadder.Length - 1); diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index 4ccc3592..17b513ac 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -73,6 +73,14 @@ public sealed class ScriptedAlarmEngine : IDisposable private readonly Func _clock; private readonly TimeSpan _scriptTimeout; + /// + /// Per-alarm run quarantine (keyed by alarm id, source identity = predicate source). Refuses a + /// predicate run while the alarm's previous (timed-out, abandoned) run is still executing and + /// quarantines after three consecutive timeouts (a refusal is not a strike unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach — one orphan + /// per alarm instead of one per upstream push. Reset on every (a redeploy). + /// + private readonly ScriptRunGuard _guard; + // ConcurrentDictionary, not a plain Dictionary: every mutation happens under // _evalGate, but four read paths (GetState, GetAllStates, LoadedAlarmIds, // RunShelvingCheck) touch _alarms from arbitrary threads (Admin UI request @@ -162,6 +170,33 @@ public sealed class ScriptedAlarmEngine : IDisposable => _scratchByAlarmId.TryGetValue(alarmId, out var s) ? s.Context : null; private readonly ConcurrentDictionary _valueCache = new(StringComparer.Ordinal); + + /// Test hook — the number of paths currently held in the value cache. + internal int ValueCacheCountForTest => _valueCache.Count; + + /// Test hook — whether is currently held in the value cache. + internal bool ValueCacheContainsForTest(string path) => _valueCache.ContainsKey(path); + + /// + /// How long waits for in-flight background work (a re-evaluation or shelving + /// check mid-SaveAsync) before cancelling it through and + /// returning. 5 s, like PollGroupEngine's drain; a hung store must not hang host shutdown. + /// + internal TimeSpan DisposeDrainTimeout { get; set; } = TimeSpan.FromSeconds(5); + + /// Cancelled by once the drain bound expires — every background task + /// (DrainDirtyAsync, ShelvingCheckAsync) runs on its token, so a store call that honours + /// cancellation returns instead of outliving the engine. + private readonly CancellationTokenSource _shutdownCts = new(); + + /// 's token, captured once: background work reads this, never the CTS, + /// so can dispose the CTS without racing a late reader. + private readonly CancellationToken _shutdownToken; + + private volatile bool _shutdownCtsDisposed; + + /// Test hook — whether disposed . + internal bool ShutdownCtsDisposedForTest => _shutdownCtsDisposed; private readonly Dictionary> _alarmsReferencing = new(StringComparer.Ordinal); // tag path -> alarm ids @@ -204,13 +239,15 @@ public sealed class ScriptedAlarmEngine : IDisposable /// The logger for engine-level diagnostics. /// Optional function providing the current UTC time; defaults to DateTime.UtcNow. /// Optional timeout for script execution; defaults to the evaluator's default timeout. + /// Run-guard budget + quarantine threshold; null ⇒ defaults. public ScriptedAlarmEngine( ITagUpstreamSource upstream, IAlarmStateStore store, ScriptLoggerFactory loggerFactory, ILogger engineLogger, Func? clock = null, - TimeSpan? scriptTimeout = null) + TimeSpan? scriptTimeout = null, + ScriptingOptions? scripting = null) { _upstream = upstream ?? throw new ArgumentNullException(nameof(upstream)); _store = store ?? throw new ArgumentNullException(nameof(store)); @@ -218,8 +255,21 @@ public sealed class ScriptedAlarmEngine : IDisposable _engineLogger = engineLogger ?? throw new ArgumentNullException(nameof(engineLogger)); _clock = clock ?? (() => DateTime.UtcNow); _scriptTimeout = scriptTimeout ?? TimedScriptEvaluator.DefaultTimeout; + _guard = new ScriptRunGuard(scripting, _engineLogger); + _shutdownToken = _shutdownCts.Token; } + /// The run guard every predicate evaluation goes through (diagnostics / tests). + public ScriptRunGuard RunGuard => _guard; + + /// + /// OPC UA BadInternalError — the quality a condition is annotated with while its predicate + /// cannot be evaluated (timed out, refused because the previous run is still executing, or + /// quarantined). The condition's state is frozen; the quality says why it cannot be trusted. The + /// next completed evaluation restores the input-derived quality through the normal bucket tracking. + /// + private const uint StatusBadInternalError = 0x80020000u; + /// Raised for every emission the Part9StateMachine produces that the engine should publish. public event EventHandler? OnEvent; @@ -259,6 +309,10 @@ public sealed class ScriptedAlarmEngine : IDisposable UnsubscribeFromUpstream(); _alarms.Clear(); _alarmsReferencing.Clear(); + // The value cache is re-seeded below for exactly the paths this generation references. It + // used to survive reloads untouched, so every path a retired alarm had ever referenced stayed + // cached forever — a slow leak over many deploys with a churning tag set. + _valueCache.Clear(); // Drop the prior generation's per-alarm scratch buffers — definitions may // have changed (different Inputs, different Logger), so any reuse would be // unsafe. @@ -268,6 +322,9 @@ public sealed class ScriptedAlarmEngine : IDisposable // recompile this one. Skipping this is what made the earlier fix a // no-op in production. _compileCache.Clear(); + // A (re)load is a deploy: lift every run-guard quarantine (an edited predicate already lifts + // its own through the guard's source identity; this covers "redeployed to retry"). + _guard.ResetAll(); var compileFailures = new List(); foreach (var def in definitions) @@ -284,15 +341,17 @@ public sealed class ScriptedAlarmEngine : IDisposable // Route through CompiledScriptCache so the emitted assembly's // collectible ALC participates in publish-replace cleanup. + // The compiled script is run through _guard (timeout + quarantine) at evaluation time — + // the per-alarm TimedScriptEvaluator this used to wrap gave the timeout but could not + // see that the run it abandoned was still executing. var evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource); - var timed = new TimedScriptEvaluator(evaluator, _scriptTimeout); var logger = _loggerFactory.Create(def.AlarmId); var templateTokens = MessageTemplate.ExtractTokenPaths(def.MessageTemplate); var allInputs = new HashSet(extraction.Reads, StringComparer.Ordinal); foreach (var t in templateTokens) allInputs.Add(t); - _alarms[def.AlarmId] = new AlarmState(def, timed, extraction.Reads, templateTokens, logger, + _alarms[def.AlarmId] = new AlarmState(def, evaluator, extraction.Reads, templateTokens, logger, AlarmConditionState.Fresh(def.AlarmId, _clock())); foreach (var path in allInputs) @@ -546,8 +605,8 @@ public sealed class ScriptedAlarmEngine : IDisposable /// The new data value snapshot. internal void OnUpstreamChange(string path, DataValueSnapshot value) { + if (_disposed) return; // a disposed engine neither caches nor queues — nothing reads it again _valueCache[path] = value; - if (_disposed) return; // don't queue new work against a disposing engine if (!_alarmsReferencing.TryGetValue(path, out var alarmIds)) return; bool startDrain; @@ -562,7 +621,7 @@ public sealed class ScriptedAlarmEngine : IDisposable // Started outside the lock: DrainDirtyAsync runs synchronously up to its // first real await, and nothing it does should happen under _dirtyLock. - if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(CancellationToken.None)); + if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(_shutdownToken)); } /// @@ -705,6 +764,11 @@ public sealed class ScriptedAlarmEngine : IDisposable } finally { _evalGate.Release(); } } + catch (OperationCanceledException) when (_shutdownToken.IsCancellationRequested) + { + // Dispose cut the drain short after its bound — expected, not a failure. + return; + } catch (Exception ex) { _engineLogger.Error(ex, "ScriptedAlarmEngine reevaluate failed"); @@ -744,7 +808,6 @@ public sealed class ScriptedAlarmEngine : IDisposable // short-circuit so an outright-Bad input is still observed. A bucket change with no state // transition is delivered out of band as a QualityChanged emission (see below). var worstStatus = WorstInputStatus(scratch.ReadCache); - var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus); // Cold-start guard — skip the predicate when any referenced upstream tag has no // cached value yet (the upstream subscription hasn't delivered its first push). @@ -757,7 +820,7 @@ public sealed class ScriptedAlarmEngine : IDisposable // The condition is frozen (can't trust its state), but its source quality just changed // bucket — annotate it out of band so a comms-lost / Bad-input scripted condition reports // Bad, mirroring the native OT path. - if (qualityBucketChanged) + if (TrackQualityBucket(state.Definition.AlarmId, worstStatus)) pendingEmissions.Add(BuildQualityEmission(state, seed, worstStatus)); return seed; } @@ -767,22 +830,41 @@ public sealed class ScriptedAlarmEngine : IDisposable bool predicateTrue; try { - predicateTrue = await state.Evaluator.RunAsync(context, ct).ConfigureAwait(false); + predicateTrue = await _guard.RunAsync( + state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator, context, + _scriptTimeout, ct) + .ConfigureAwait(false); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } catch (ScriptTimeoutException tex) { - state.Logger.Warning("Alarm predicate timed out after {Timeout} — state unchanged", tex.Timeout); - return seed; + return PredicateDidNotComplete(state, seed, scratch, + $"timed out after {tex.Timeout.TotalMilliseconds:F0} ms", pendingEmissions); + } + catch (ScriptRunRefusedException ex) + { + return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions); + } + catch (ScriptQuarantinedException ex) + { + return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions); } catch (Exception ex) { + scratch.LastNonCompletion = null; state.Logger.Error(ex, "Alarm predicate threw — state unchanged"); return seed; } + scratch.LastNonCompletion = null; + + // Track the bucket only once the predicate has COMPLETED (I3). The tracked bucket is the last + // bucket this alarm was annotated with; tracking the input bucket before the attempt flipped it to + // Good on every push, so a refused / quarantined predicate (which annotates Bad) re-emitted + // QualityChanged on every push instead of once per real bucket change. + var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus); var result = Part9StateMachine.ApplyPredicate(seed, predicateTrue, nowUtc); var transition = result.Emission != EmissionKind.None @@ -802,6 +884,28 @@ public sealed class ScriptedAlarmEngine : IDisposable return result.State; } + /// + /// The predicate did not return (timed out, refused because the previous run is still executing, or + /// quarantined): hold the condition at , log the reason once per change of + /// reason (a quarantined alarm is re-attempted on every push and would otherwise log every time), + /// and annotate the condition out of band — the frozen state + /// cannot be trusted and the quality is how the operator sees that. The next completed evaluation + /// restores the input-derived quality through the normal bucket tracking. + /// + private AlarmConditionState PredicateDidNotComplete( + AlarmState state, AlarmConditionState seed, AlarmScratch scratch, string reason, + List pendingEmissions) + { + if (!string.Equals(scratch.LastNonCompletion, reason, StringComparison.Ordinal)) + { + scratch.LastNonCompletion = reason; + state.Logger.Warning("Alarm predicate did not complete ({Reason}) — state unchanged, quality Bad", reason); + } + if (TrackQualityBucket(state.Definition.AlarmId, StatusBadInternalError)) + pendingEmissions.Add(BuildQualityEmission(state, seed, StatusBadInternalError)); + return seed; + } + /// /// Refill in place from _valueCache, falling /// back to a synchronous ITagUpstreamSource.ReadTag for paths whose @@ -986,7 +1090,7 @@ public sealed class ScriptedAlarmEngine : IDisposable { if (_disposed) return; var ids = _alarms.Keys.ToArray(); - TrackBackgroundTask(() => ShelvingCheckAsync(ids, CancellationToken.None)); + TrackBackgroundTask(() => ShelvingCheckAsync(ids, _shutdownToken)); } /// @@ -1125,6 +1229,10 @@ public sealed class ScriptedAlarmEngine : IDisposable } finally { _evalGate.Release(); } } + catch (OperationCanceledException) when (_shutdownToken.IsCancellationRequested) + { + return; // Dispose cut the check short after its bound + } catch (Exception ex) { _engineLogger.Warning(ex, "ScriptedAlarmEngine shelving-check failed"); @@ -1165,11 +1273,25 @@ public sealed class ScriptedAlarmEngine : IDisposable // flight can outlive the engine and write to a (possibly disposed) store // after Dispose() has returned. The tasks re-check _disposed after // acquiring the gate and bail out, but the await still has to complete. + // BOUNDED (R05:3): the drained tasks end in IAlarmStateStore.SaveAsync, and a stalled store + // (connection stall, lock, replication hiccup) used to hang this — and the host shutdown that + // calls it — indefinitely. Wait DisposeDrainTimeout; past it, cancel the tasks' shared token (a + // store that honours cancellation returns promptly), log, and give them one short grace period. Task[] toAwait; lock (_inFlightLock) { toAwait = [.. _inFlight]; } if (toAwait.Length > 0) { - try { Task.WhenAll(toAwait).GetAwaiter().GetResult(); } + var all = Task.WhenAll(toAwait); + try { all.WaitAsync(DisposeDrainTimeout).GetAwaiter().GetResult(); } + catch (TimeoutException) + { + _engineLogger.Warning( + "ScriptedAlarmEngine dispose: {Count} background task(s) still running after {Timeout} — cancelling them (a store call is stalled)", + toAwait.Length, DisposeDrainTimeout); + _shutdownCts.Cancel(); + try { all.WaitAsync(TimeSpan.FromSeconds(1)).GetAwaiter().GetResult(); } + catch (Exception) { /* cancelled, faulted, or still stuck — we are leaving either way */ } + } catch (Exception ex) { // Background task failures already logged inside ReevaluateAsync / @@ -1192,6 +1314,16 @@ public sealed class ScriptedAlarmEngine : IDisposable _alarmsReferencing.Clear(); _scratchByAlarmId.Clear(); _lastQualityBucketByAlarmId.Clear(); + + // Release the shutdown CTS once nothing can still be observing it through a registration: every + // drained task has completed. A task still stuck past the grace keeps its registration, so the CTS is + // left to the GC in that case (logged above). + if (toAwait.All(t => t.IsCompleted)) + { + _shutdownCts.Dispose(); + _shutdownCtsDisposed = true; + } + // Dispose every compiled-predicate ALC so the engine's shutdown actually // releases the emitted assemblies. The drain above ensures no evaluator is // mid-call; CompiledScriptCache.Dispose internally guards against use-after- @@ -1201,7 +1333,7 @@ public sealed class ScriptedAlarmEngine : IDisposable private sealed record AlarmState( ScriptedAlarmDefinition Definition, - TimedScriptEvaluator Evaluator, + ScriptEvaluator Evaluator, IReadOnlySet Inputs, IReadOnlyList TemplateTokens, ILogger Logger, @@ -1227,6 +1359,10 @@ public sealed class ScriptedAlarmEngine : IDisposable /// Gets the predicate evaluation context. public AlarmPredicateContext Context { get; } + /// The reason the last evaluation did not complete (timeout / refused / quarantined), or null + /// after a completed run — the once-per-change log gate for PredicateDidNotComplete. + public string? LastNonCompletion { get; set; } + /// /// Initializes a new AlarmScratch with the specified inputs, logger, and clock. /// diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs index dc76923c..9d44aee5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using Serilog; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; @@ -23,19 +24,27 @@ namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; /// adapter doesn't maintain per-subscription state beyond the filter set — it /// checks each emission against every live subscription. /// +/// +/// A throwing subscriber is logged at Warning (naming the alarm) and +/// does not crash the adapter — the same posture as the engine's own FireEvent. It used to +/// be swallowed silently: a lost alarm event with no trace. +/// /// public sealed class ScriptedAlarmSource : IAlarmSource, IDisposable { private readonly ScriptedAlarmEngine _engine; + private readonly ILogger _logger; private readonly ConcurrentDictionary _subscriptions = new(StringComparer.Ordinal); private bool _disposed; /// Initializes a new instance of the class. /// The scripted alarm engine to expose. - public ScriptedAlarmSource(ScriptedAlarmEngine engine) + /// Where a throwing subscriber is reported; null ⇒ Serilog's global logger. + public ScriptedAlarmSource(ScriptedAlarmEngine engine, ILogger? logger = null) { _engine = engine ?? throw new ArgumentNullException(nameof(engine)); + _logger = logger ?? Log.Logger; _engine.OnEvent += OnEngineEvent; } @@ -98,7 +107,13 @@ public sealed class ScriptedAlarmSource : IAlarmSource, IDisposable Severity: evt.Severity, SourceTimestampUtc: evt.TimestampUtc); try { OnAlarmEvent?.Invoke(this, payload); } - catch { /* subscriber exceptions don't crash the adapter */ } + catch (Exception ex) + { + // Subscriber exceptions don't crash the adapter — but they are not silent either: a lost + // alarm event must leave a trace the operator can find. + _logger.Warning(ex, "ScriptedAlarmSource OnAlarmEvent subscriber threw for {AlarmId} ({Emission})", + evt.AlarmId, evt.Emission); + } } } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs index 005734b6..5f7d8140 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs @@ -17,13 +17,15 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// /// /// -/// Deny-list is the authoritative Phase 7 plan set: +/// Deny-list is the authoritative Phase 7 plan set, hardened since: /// System.IO, System.Net, System.Diagnostics, -/// System.Reflection, System.Threading.Thread, -/// System.Threading.Tasks (scripts are synchronous predicates — no -/// legitimate need to start background tasks; a Task.Run fan-out outlives -/// the evaluation timeout entirely), System.Runtime.InteropServices, -/// Microsoft.Win32. +/// System.Reflection, System.Threading.Tasks and the whole of +/// System.Threading (scripts are synchronous predicates — no legitimate need +/// to start background work, and no blocking primitive may park the evaluation's pool +/// thread past its budget; CancellationToken / Interlocked / +/// Volatile are allow-listed back in), System.Runtime.InteropServices, +/// System.Runtime.Loader, Microsoft.Win32; plus the member-granular +/// (System.Type's reflection entry points). /// /// /// Deny-list prefix match. System.Net catches System.Net.Http, @@ -63,13 +65,19 @@ public static class ForbiddenTypeAnalyzer "System.Net", "System.Diagnostics", // catches Process, ProcessStartInfo, EventLog, Trace/Debug file sinks "System.Reflection", - // System.Threading.Thread is NOT in this list: Thread's containing namespace is - // "System.Threading" (not "System.Threading.Thread"), so a prefix check on - // "System.Threading.Thread" never matches. Thread is denied type-granularly via - // ForbiddenFullTypeNames instead so the check actually fires. "System.Threading.Tasks", // Task.Run / Parallel — scripts are synchronous predicates // and have no legitimate need to start background work; // a Task fan-out outlives the evaluation timeout entirely. + // Listed BEFORE the parent prefix so its rejection names it. + "System.Threading", // Thread / ThreadPool / Timer (background work that outlives + // the timeout) AND every blocking primitive — SemaphoreSlim, + // Monitor, SpinWait, ManualResetEvent(Slim), Mutex, + // EventWaitHandle — which park the script's pool thread past + // its budget (the same orphan ScriptRunGuard exists to bound) + // and, for the named kernel objects, touch system-global + // state. The few sync-free primitives a script may + // legitimately name are allow-listed in + // AllowedTypesInForbiddenNamespaces. "System.Runtime.InteropServices", "System.Runtime.Loader", // AssemblyLoadContext + AssemblyDependencyResolver — // arbitrary DLL load into the host process. @@ -79,11 +87,55 @@ public static class ForbiddenTypeAnalyzer "Microsoft.Win32", // registry ]; + /// + /// Types that stay reachable although their namespace is in + /// . Matched by exact fully-qualified name against the + /// resolved type symbol (generic constructions unwrapped). The whole of System.Threading is + /// denied; these three are the sync-free primitives with no blocking surface — a + /// is read-only state, and + /// are single-instruction memory operations. + /// + public static readonly IReadOnlyList AllowedTypesInForbiddenNamespaces = + [ + "System.Threading.CancellationToken", + "System.Threading.Interlocked", + "System.Threading.Volatile", + ]; + + /// + /// Members scripts are NOT allowed to call on otherwise-allowed types, as + /// (declaring type, member name). System.Type must stay reachable + /// (typeof, Name, IsValueType are ordinary metadata reads) but its reflection + /// entry points obtain a type or member by NAME — Type.GetType("System.IO.File") reaches a + /// forbidden type without ever naming it syntactically, and GetMethod* / GetProperty + /// / InvokeMember / … hand back a System.Reflection handle (which pass 2 catches when + /// declared, but not when consumed inline). Instance object.GetType() is declared on + /// System.Object and is not matched. + /// + public static readonly IReadOnlyList<(string Type, string Member)> ForbiddenMembers = + [ + ("System.Type", "GetType"), + ("System.Type", "GetMethod"), + ("System.Type", "GetMethods"), + ("System.Type", "InvokeMember"), + ("System.Type", "GetProperty"), + ("System.Type", "GetProperties"), + ("System.Type", "GetField"), + ("System.Type", "GetFields"), + ("System.Type", "GetMember"), + ("System.Type", "GetMembers"), + ("System.Type", "GetConstructor"), + ("System.Type", "GetConstructors"), + ("System.Type", "GetEvent"), + ("System.Type", "GetEvents"), + ("System.Type", "GetDefaultMembers"), + ]; + /// /// Fully-qualified type names scripts are NOT allowed to reference, regardless of /// namespace. Used when a dangerous type shares its namespace with legitimate types - /// (e.g. System.Threading hosts both CancellationToken and - /// Thread) so a namespace-prefix rule would over-block. Matched by exact + /// (e.g. System.Runtime.CompilerServices hosts both MethodImplOptions and + /// RuntimeHelpers) so a namespace-prefix rule would over-block. Matched by exact /// fully-qualified name against the resolved type symbol — every member of /// the type (including read-only ones) is therefore rejected. /// @@ -99,19 +151,22 @@ public static class ForbiddenTypeAnalyzer /// System.Activator — CreateInstance is a /// reflection-equivalent escape that constructs a forbidden type by name /// without ever naming it syntactically. - /// System.Threading.Thread — raw thread creation bypasses the - /// per-evaluation timeout; denied type-granularly because its containing - /// namespace is System.Threading (shared with allowed types like - /// CancellationToken), so a namespace-prefix rule cannot reach it - /// without blocking unrelated types. - /// System.Threading.ThreadPool / System.Threading.Timer — - /// background-work vectors that outlive the per-evaluation timeout; same - /// threat as Task.Run. /// System.Runtime.CompilerServices.Unsafe — exposes raw type- /// reinterpretation (As<TFrom,TTo>, As<T>(object)) /// that bypasses the CLR type system without requiring an unsafe context at /// the call site; enables type-confusion and managed heap corruption. + /// System.Runtime.CompilerServices.RuntimeHelpers — + /// GetUninitializedObject constructs an instance of any type (a forbidden + /// one included) with no constructor and no syntactic type name beyond a + /// Type value; PrepareMethod / RunClassConstructor reach + /// runtime internals. /// + /// + /// System.Threading.Thread / ThreadPool / Timer used to be listed + /// here; since the whole System.Threading namespace is denied (with a type-granular + /// allow-list, see ) the prefix rule reaches + /// them first. + /// /// public static readonly IReadOnlyList ForbiddenFullTypeNames = [ @@ -119,26 +174,6 @@ public static class ForbiddenTypeAnalyzer "System.AppDomain", "System.GC", "System.Activator", - // System.Threading.Thread lives in the System.Threading namespace (shared with - // CancellationToken, SemaphoreSlim, etc.), so a namespace-prefix deny-list cannot - // target it without blocking those legitimate types. Denied type-granularly here. - "System.Threading.Thread", - // Broadening the references list to the BCL trusted-platform-assemblies set - // re-exposed two background-work vectors the original deny-list missed. Both live - // in System.Threading (shared with allowed sync primitives like CancellationToken / - // SemaphoreSlim), so they must be denied type-granularly: - // - // System.Threading.ThreadPool — QueueUserWorkItem / UnsafeQueueUserWorkItem - // re-introduce the background-fanout threat - // closed against System.Threading.Tasks. - // System.Threading.Timer — Timer(callback, …) schedules unbounded work - // that outlives the per-evaluation timeout. - // - // System.Runtime.Loader.AssemblyLoadContext is also covered, but at the namespace- - // prefix level above (System.Runtime.Loader) so future BCL additions to that - // namespace are denied by default. - "System.Threading.ThreadPool", - "System.Threading.Timer", // System.Runtime.CompilerServices.Unsafe exposes raw type-reinterpretation // (Unsafe.As, Unsafe.As(object)) that bypasses the CLR type system // without requiring an 'unsafe' compilation context at the call site. In a script, @@ -148,6 +183,11 @@ public static class ForbiddenTypeAnalyzer // compile-time-only types (CallerMemberNameAttribute, MethodImplAttribute, // MethodImplOptions) that scripts may legitimately reference. "System.Runtime.CompilerServices.Unsafe", + // RuntimeHelpers.GetUninitializedObject(Type) builds an instance of ANY type — a + // forbidden one obtained through Type.GetType (itself denied via ForbiddenMembers) + // included — without a constructor or a syntactic type name. Same namespace caveat + // as Unsafe above, so type-granular. + "System.Runtime.CompilerServices.RuntimeHelpers", ]; /// @@ -277,28 +317,61 @@ public static class ForbiddenTypeAnalyzer if (rejections.Any(r => r.Span == span && r.TypeName == typeName)) return; - var ns = typeSymbol.ContainingNamespace?.ToDisplayString() ?? string.Empty; - foreach (var forbidden in ForbiddenNamespacePrefixes) - { - if (ns == forbidden || ns.StartsWith(forbidden + ".", StringComparison.Ordinal)) - { - rejections.Add(new ForbiddenTypeRejection( - Span: span, - TypeName: typeName, - Namespace: ns, - Message: $"Type '{typeName}' is in the forbidden namespace '{ns}'. " + - $"Scripts cannot reach {forbidden}* per Phase 7 sandbox rules.")); - return; - } - } - - // Type-granular deny-list — dangerous types that live in the allow-listed - // System namespace and so cannot be caught by ForbiddenNamespacePrefixes. - // Matched on the full type name; OriginalDefinition unwraps any generic - // construction before naming. + // Full name of the (unwrapped) type — shared by the allow-list, the type deny-list and the + // member deny-list below. OriginalDefinition unwraps any generic construction before naming. var fullTypeName = typeSymbol.OriginalDefinition.ToDisplayString( SymbolDisplayFormat.FullyQualifiedFormat.WithGlobalNamespaceStyle( SymbolDisplayGlobalNamespaceStyle.Omitted)); + + var ns = typeSymbol.ContainingNamespace?.ToDisplayString() ?? string.Empty; + + // Member deny-list first: the containing type (System.Type) is allowed, so neither list + // below would ever look at the member name. + if (symbol is IMethodSymbol method) + { + var declaring = method.ContainingType.OriginalDefinition.ToDisplayString( + SymbolDisplayFormat.FullyQualifiedFormat.WithGlobalNamespaceStyle( + SymbolDisplayGlobalNamespaceStyle.Omitted)); + foreach (var (type, member) in ForbiddenMembers) + { + if (declaring == type && method.Name == member) + { + rejections.Add(new ForbiddenTypeRejection( + Span: span, + TypeName: $"{type}.{member}", + Namespace: ns, + Message: $"Member '{type}.{member}' is on the sandbox forbidden-member deny-list: " + + "it obtains a type or member by name, which bypasses the type allow-list " + + "(see ForbiddenTypeAnalyzer.ForbiddenMembers for the authoritative list).")); + return; + } + } + } + + // A type on the allow-list stays reachable even though its namespace is denied; it is still + // subject to the type-granular deny-list below (nothing is on both today). + var allowListed = AllowedTypesInForbiddenNamespaces.Contains(fullTypeName, StringComparer.Ordinal); + + if (!allowListed) + { + foreach (var forbidden in ForbiddenNamespacePrefixes) + { + if (ns == forbidden || ns.StartsWith(forbidden + ".", StringComparison.Ordinal)) + { + rejections.Add(new ForbiddenTypeRejection( + Span: span, + TypeName: typeName, + Namespace: ns, + Message: $"Type '{typeName}' is in the forbidden namespace '{ns}'. " + + $"Scripts cannot reach {forbidden}* per Phase 7 sandbox rules.")); + return; + } + } + } + + // Type-granular deny-list — dangerous types that live in an allow-listed namespace + // (System, System.Runtime.CompilerServices) and so cannot be caught by + // ForbiddenNamespacePrefixes. foreach (var forbiddenType in ForbiddenFullTypeNames) { if (fullTypeName == forbiddenType) diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs new file mode 100644 index 00000000..3cf38fac --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs @@ -0,0 +1,15 @@ +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Implemented by a script seam that runs its scripts through a , so the +/// host that drives its deploy boundary can lift every quarantine on a redeploy (the guard's own +/// source-change detection covers an edited script; this covers "the operator redeployed to retry"). +/// +public interface IScriptQuarantineOwner +{ + /// Lift every quarantine and forget every consecutive-timeout count (). + void ResetQuarantines(); + + /// The script keys currently quarantined. + IReadOnlyCollection QuarantinedScripts { get; } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs index fb1aff35..84bb0e04 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs @@ -24,22 +24,32 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// Original exception (if any) is attached so the main log's diagnostics keep /// the full stack trace. /// +/// +/// Rate-limited per script. A predicate that throws on every push would otherwise mirror +/// one Warning per push into the main log; a keyed by script +/// name admits 20 per minute and mirrors one summary line for the rest when the window rolls. The +/// scripts-*.log file sink is unaffected and keeps the full stream. +/// /// public sealed class ScriptLogCompanionSink : ILogEventSink { private readonly ILogger _mainLogger; private readonly LogEventLevel _minMirrorLevel; + private readonly ScriptLogRateLimiter _limiter; /// Initializes a new instance of the ScriptLogCompanionSink. /// The companion logger to mirror events to. /// The minimum log level to mirror (default: Error). - public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error) + /// Per-script rate limiter; null ⇒ the default 20 per minute. + public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error, + ScriptLogRateLimiter? limiter = null) { _mainLogger = mainLogger ?? throw new ArgumentNullException(nameof(mainLogger)); _minMirrorLevel = minMirrorLevel; + _limiter = limiter ?? new ScriptLogRateLimiter(); } - /// Emits a log event to the companion logger if it meets the minimum level. + /// Emits a log event to the companion logger if it meets the minimum level and the script is within its per-minute budget. /// The log event to emit. public void Emit(LogEvent logEvent) { @@ -54,6 +64,15 @@ public sealed class ScriptLogCompanionSink : ILogEventSink scriptName = s; } + var admission = _limiter.Admit(scriptName); + if (admission.SuppressedInPreviousWindow > 0) + { + _mainLogger.Warning( + "[Script] {ScriptName}: {SuppressedCount} further script error(s) were suppressed in the previous {WindowSeconds:F0} s; see scripts-*.log", + scriptName, admission.SuppressedInPreviousWindow, _limiter.Window.TotalSeconds); + } + if (!admission.Admitted) return; + var rendered = logEvent.RenderMessage(); if (logEvent.Exception is not null) { diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs new file mode 100644 index 00000000..5cce5fcd --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs @@ -0,0 +1,126 @@ +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Per-source fixed-window rate limit for the script-log fan-out: the first +/// entries a source emits in a are +/// admitted, the rest are counted and dropped, and the count is handed back exactly once — on the first +/// admission after the window rolls — so the caller can emit one summary line in the dropped entries' +/// place. Nothing is silently lost (the operator sees "N suppressed"), and one chatty script cannot +/// shed every other script's entries out of the bounded telemetry channels (drop-oldest) or flood the +/// cluster bus / main log. +/// +/// +/// Used by (key = script + virtual-tag + alarm identity), +/// (key = script name) and the virtual-tag actor's per-failure +/// log (one key per tag). Thread-safe; Serilog invokes a sink on whichever thread logged. Idle sources +/// are swept every admissions so a churning tag set cannot grow the table +/// without bound. +/// +public sealed class ScriptLogRateLimiter +{ + /// Entries admitted per source per window by default: 20. + public const int DefaultLimitPerWindow = 20; + + /// The default window: one minute. + public static readonly TimeSpan DefaultWindow = TimeSpan.FromMinutes(1); + + private const int SweepEvery = 256; + + private readonly Dictionary _sources = new(StringComparer.Ordinal); + private readonly int _limit; + private readonly TimeSpan _window; + private readonly Func _clock; + private int _admitsSinceSweep; + + /// Initializes a limiter. + /// Entries admitted per source per window; values below 1 are treated as 1. + /// The fixed window; null ⇒ . + /// UTC clock; null ⇒ . + public ScriptLogRateLimiter(int limitPerWindow = DefaultLimitPerWindow, TimeSpan? window = null, Func? clock = null) + { + _limit = Math.Max(1, limitPerWindow); + _window = window is { } w && w > TimeSpan.Zero ? w : DefaultWindow; + _clock = clock ?? (static () => DateTime.UtcNow); + } + + /// Entries admitted per source per window, as configured (after clamping to at least 1). + public int LimitPerWindow => _limit; + + /// The window length. + public TimeSpan Window => _window; + + /// + /// Ask whether 's next entry may be emitted. When the window has rolled + /// since the last call, carries the + /// number of entries dropped in the window that just closed (zero otherwise) — the caller emits one + /// summary for them BEFORE the admitted entry. + /// + /// Identity of the emitting script instance. + /// The admission decision. + public ScriptLogAdmission Admit(string sourceKey) + { + ArgumentNullException.ThrowIfNull(sourceKey); + var now = _clock(); + lock (_sources) + { + if (++_admitsSinceSweep >= SweepEvery) + { + _admitsSinceSweep = 0; + Sweep(now); + } + + if (!_sources.TryGetValue(sourceKey, out var source)) + { + source = new Source { WindowStart = now }; + _sources[sourceKey] = source; + } + + long summary = 0; + if (now - source.WindowStart >= _window) + { + summary = source.Suppressed; + source.WindowStart = now; + source.Admitted = 0; + source.Suppressed = 0; + } + + source.LastSeen = now; + if (source.Admitted < _limit) + { + source.Admitted++; + return new ScriptLogAdmission(true, summary); + } + + source.Suppressed++; + return new ScriptLogAdmission(false, summary); + } + } + + private void Sweep(DateTime now) + { + var idle = now - (_window + _window); + List? dead = null; + foreach (var (key, source) in _sources) + { + // A source that still has a pending summary is kept: its next entry must report the count. + if (source.LastSeen < idle && source.Suppressed == 0) + (dead ??= []).Add(key); + } + if (dead is null) return; + foreach (var key in dead) _sources.Remove(key); + } + + private sealed class Source + { + public DateTime WindowStart; + public DateTime LastSeen; + public int Admitted; + public long Suppressed; + } +} + +/// Result of . +/// Whether the entry may be emitted. +/// Entries dropped in the window that just closed — non-zero at +/// most once per window roll; the caller emits one summary line for them. +public readonly record struct ScriptLogAdmission(bool Admitted, long SuppressedInPreviousWindow); diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs index fe9e7f3e..88bdd973 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs @@ -25,25 +25,37 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// property so pipelines that pre-date this sink continue to work correctly. If /// neither property is present ScriptId defaults to "unknown". /// +/// +/// Rate-limited per source. A script that logs at Information on every evaluation of a 1 Hz +/// input put one rendered string per second per script onto the cluster bus; 200 such scripts shed +/// everyone's entries out of the bounded telemetry channels. A +/// keyed by the entry's identity (script + virtual tag + alarm) admits 20 entries per minute per +/// source and publishes one summary entry naming the dropped count when the window rolls. +/// /// public sealed class ScriptLogTopicSink : ILogEventSink { private readonly IScriptLogPublisher _publisher; private readonly LogEventLevel _min; + private readonly ScriptLogRateLimiter _limiter; /// Initializes a new instance of the class. /// The publisher that routes entries to the cluster bus. Must not be null. /// Minimum log level to forward. Events below this level are discarded (default: ). + /// Per-source rate limiter; null ⇒ the default 20 per minute. /// Thrown when is null. - public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information) + public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information, + ScriptLogRateLimiter? limiter = null) { _publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); _min = min; + _limiter = limiter ?? new ScriptLogRateLimiter(); } /// /// Converts the to a and - /// publishes it, unless the event is below the configured minimum level. + /// publishes it, unless the event is below the configured minimum level or the source is over its + /// per-minute budget. /// /// The Serilog event to process. Silently ignored when null. public void Emit(LogEvent logEvent) @@ -56,15 +68,36 @@ public sealed class ScriptLogTopicSink : ILogEventSink ? s : null; + var scriptId = P(ScriptLoggerFactory.ScriptIdProperty) + ?? P(ScriptLoggerFactory.ScriptNameProperty) + ?? "unknown"; + var virtualTagId = P(ScriptLoggerFactory.VirtualTagIdProperty); + var alarmId = P(ScriptLoggerFactory.AlarmIdProperty); + var equipmentId = P(ScriptLoggerFactory.EquipmentIdProperty); + + var admission = _limiter.Admit(string.Concat(scriptId, "|", virtualTagId, "|", alarmId)); + if (admission.SuppressedInPreviousWindow > 0) + { + _publisher.Publish(new ScriptLogEntry( + ScriptId: scriptId, + Level: nameof(LogEventLevel.Warning), + Message: $"{admission.SuppressedInPreviousWindow} script-log entr{(admission.SuppressedInPreviousWindow == 1 ? "y" : "ies")} " + + $"from this source were suppressed in the previous {_limiter.Window.TotalSeconds:F0} s " + + $"(limit {_limiter.LimitPerWindow}/window); see scripts-*.log for the full stream", + TimestampUtc: logEvent.Timestamp.UtcDateTime, + VirtualTagId: virtualTagId, + AlarmId: alarmId, + EquipmentId: equipmentId)); + } + if (!admission.Admitted) return; + _publisher.Publish(new ScriptLogEntry( - ScriptId: P(ScriptLoggerFactory.ScriptIdProperty) - ?? P(ScriptLoggerFactory.ScriptNameProperty) - ?? "unknown", + ScriptId: scriptId, Level: logEvent.Level.ToString(), Message: logEvent.RenderMessage(), TimestampUtc: logEvent.Timestamp.UtcDateTime, - VirtualTagId: P(ScriptLoggerFactory.VirtualTagIdProperty), - AlarmId: P(ScriptLoggerFactory.AlarmIdProperty), - EquipmentId: P(ScriptLoggerFactory.EquipmentIdProperty))); + VirtualTagId: virtualTagId, + AlarmId: alarmId, + EquipmentId: equipmentId)); } } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs new file mode 100644 index 00000000..de1afee2 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs @@ -0,0 +1,420 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Serilog; +using ZB.MOM.WW.OtOpcUa.Commons.Observability; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Per-script run quarantine shared by every script seam on the node. Wraps each evaluation of a +/// with the wall-clock timeout +/// used to provide, and adds what that wrapper +/// could not: knowledge of whether the script it abandoned is still running. +/// +/// +/// +/// The leak this closes. A timed-out script is abandoned, not stopped — a CPU-bound +/// while(true) keeps its thread-pool thread forever. Each seam re-evaluated on every trigger, +/// so a runaway script on a 1 Hz input leaked one pool thread per second until the pool starved and +/// unrelated work (Akka dispatchers, the OPC UA publish thread, gRPC) stalled. The guard keys every +/// run by script instance and: +/// +/// +/// Refuses to start a run while that script's previous run is still executing — +/// (), counted on +/// . One orphan per script, never one per trigger. +/// Quarantines after +/// consecutive timeouts (only a timeout is a strike; a refusal is a Bad publish plus its counter +/// and leaves the streak untouched — on a 1 Hz trigger with a 2 s budget one 4 s overrun is a +/// timeout plus two refusals and must not quarantine; a run that returns — a value or a throw — +/// resets the streak), or on a refusal behind a stuck orphan — one that has been executing for +/// longer than × the timeout (20 s at the +/// 2 s default), which is how a never-returning while(true) (one timeout, then only refusals) +/// gets quarantined: no further dispatch, every attempt throws +/// , one Error log, one otopcua.script.quarantined +/// increment, one event. Lifted by a source change (the operator fixed the +/// script — a new source identity under the same key starts clean and the old orphan no longer blocks +/// it), by on redeploy, or by . +/// Quarantines on an allocation-budget breach: the run's own-thread allocation is sampled +/// with around the script delegate and compared +/// with . This is observation after the fact — a +/// single run that allocates enough to OOM the process is not stopped; that needs an out-of-process +/// runner. +/// +/// +/// The timeout clock starts when the script starts, not when it is queued. The run is pushed to +/// the pool (a CPU-bound script would otherwise execute inline and return an already-completed task +/// that no timeout can fire on), and the wait only begins once the pool has actually picked the run +/// up — so thread-pool starvation is charged to nobody's script. A caller-supplied cancellation wins +/// over the timeout (, never +/// ) so shutdown paths are not misread as timeouts. +/// +/// +/// Cancellation reaches the evaluator, not (yet) the script. On timeout the token handed to +/// is cancelled; a script can only observe +/// it once ScriptContext exposes it (follow-up — the context type lives in the Abstractions +/// project). +/// +/// +public sealed class ScriptRunGuard +{ + /// Failure reason a seam publishes when a run was refused because the previous one is still executing. + public const string PreviousRunStillRunningReason = "previous evaluation still running"; + + /// Prefix of every quarantine reason / message. + public const string QuarantinedReasonPrefix = "script quarantined: "; + + /// + /// Counter otopcua.script.quarantined (Prometheus: otopcua_script_quarantined): one + /// increment per script quarantined, tagged reason = timeouts | stuck | allocation. + /// + public static readonly Counter QuarantinedCounter = + OtOpcUaTelemetry.Meter.CreateCounter("otopcua.script.quarantined", unit: "{script}", + description: "Scripts quarantined by the run guard (reason: timeouts | stuck | allocation)."); + + /// + /// Counter otopcua.script.run_refused: runs refused because the script's previous run was + /// still executing. + /// + public static readonly Counter RefusedCounter = + OtOpcUaTelemetry.Meter.CreateCounter("otopcua.script.run_refused", unit: "{run}", + description: "Script runs refused because the previous run of the same script was still executing."); + + private readonly ConcurrentDictionary _keys = new(StringComparer.Ordinal); + private readonly long _maxAllocBytesPerRun; + private readonly int _quarantineThreshold; + private readonly double _stuckFactor; + private readonly ILogger _logger; + private int _inFlight; + private long _started; + private long _refused; + private long _quarantined; + + /// Initializes a new guard. + /// Budget + threshold; null ⇒ defaults. + /// Where the quarantine Error line goes; null ⇒ Serilog's global logger. + public ScriptRunGuard(ScriptingOptions? options = null, ILogger? logger = null) + { + options ??= new ScriptingOptions(); + _maxAllocBytesPerRun = options.MaxAllocBytesPerRun; + _quarantineThreshold = Math.Max(1, options.QuarantineAfterConsecutiveTimeouts); + _stuckFactor = options.StuckOrphanQuarantineFactor; + _logger = logger ?? Log.Logger; + } + + /// Raised (on the thread that detected it) when a script is quarantined. + public event EventHandler? Quarantined; + + /// Runs currently executing on a pool thread — abandoned (timed-out) orphans included. + public int InFlightCount => Volatile.Read(ref _inFlight); + + /// Runs actually dispatched to the pool, lifetime. + public long StartedRunCount => Interlocked.Read(ref _started); + + /// Runs refused because the same script's previous run was still executing, lifetime. + public long RefusedWhileRunningCount => Interlocked.Read(ref _refused); + + /// Scripts quarantined, lifetime (a reset and re-quarantine counts twice). + public long QuarantinedCount => Interlocked.Read(ref _quarantined); + + /// The keys currently quarantined. + public IReadOnlyCollection QuarantinedKeys => + _keys.Where(kv => kv.Value.QuarantineReason is not null).Select(kv => kv.Key).ToArray(); + + /// Whether is quarantined, and why. + /// The script instance key. + /// The quarantine reason (prefixed ), or null. + /// true when quarantined. + public bool IsQuarantined(string key, out string? reason) + { + reason = _keys.TryGetValue(key, out var state) ? state.QuarantineReason : null; + return reason is not null; + } + + /// + /// Forget everything about : clears its quarantine and its consecutive count, and + /// abandons its in-flight slot (an orphan still running keeps counting on + /// until it finishes, but no longer blocks the key). + /// + /// The script instance key. + public void Reset(string key) => _keys.TryRemove(key, out _); + + /// + /// for every key — the redeploy / compile-cache-clear boundary. Orphans keep + /// counting on until they finish. + /// + public void ResetAll() => _keys.Clear(); + + /// + /// Run against under the guard. + /// + /// The script context type. + /// The script result type. + /// The script instance key (virtual-tag id, calc-tag RawPath, alarm id). + /// + /// The script source (or its hash). A change under the same key clears the key's quarantine and + /// abandons its orphan slot — the operator fixed the script. Null ⇒ not tracked. + /// + /// The compiled script. + /// The script's ctx. + /// Wall-clock budget, measured from the moment the pool starts the run. + /// Caller cancellation; wins over the timeout. + /// The script's result. + /// The key is (or just became) quarantined. + /// The key's previous run is still executing. + /// The run exceeded . + /// was cancelled. + public async Task RunAsync( + string key, + string? sourceIdentity, + ScriptEvaluator evaluator, + TContext context, + TimeSpan timeout, + CancellationToken ct = default) + where TContext : ScriptContext + { + ArgumentNullException.ThrowIfNull(key); + ArgumentNullException.ThrowIfNull(evaluator); + ArgumentNullException.ThrowIfNull(context); + if (timeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(timeout), "Timeout must be positive."); + + var state = Admit(key, sourceIdentity); + + // Slot acquisition is the whole mechanism: a second trigger landing while the previous run (orphan or + // not) still holds the slot is refused here, before anything is dispatched to the pool. + if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0) + { + // A refusal is a Bad publish plus its counter — NOT a strike. Only a timeout (and an allocation + // breach) counts toward quarantine: on a 1 Hz trigger with a 2 s budget a single 4 s overrun is + // one timeout followed by two refusals, and counting those would quarantine a script that merely + // ran long once. The refusal also leaves the timeout streak untouched. + Interlocked.Increment(ref _refused); + RefusedCounter.Add(1, new KeyValuePair("script", key)); + // …unless the orphan is STUCK: a never-returning script times out once and is then only ever + // refused, so the timeout streak alone would never quarantine it. Past the stuck threshold the + // refusal itself quarantines (same path: Error log, metric, Bad, lifted on redeploy/source change). + if (_stuckFactor > 0 && Volatile.Read(ref state.RunStartedTimestamp) is var startedAt and not 0) + { + var running = Stopwatch.GetElapsedTime(startedAt); + var threshold = timeout * _stuckFactor; + if (running > threshold) + { + Quarantine(key, state, + $"its previous evaluation has been running for {running.TotalSeconds:F1} s, over " + + $"{_stuckFactor:0.##}x its {timeout.TotalMilliseconds:F0} ms budget " + + "(Scripting:StuckOrphanQuarantineFactor) — it is stuck; the script is not evaluated again " + + "until its source changes or it is redeployed", + "stuck"); + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + } + } + throw new ScriptRunRefusedException(key); + } + + Interlocked.Increment(ref _inFlight); + Interlocked.Increment(ref _started); + + var slot = new RunSlot(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var runCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + var runToken = runCts.Token; + + // Task.Run deliberately takes NO token: a pre-cancelled token would skip the lambda, `started` would + // never complete, and the slot would never be released. The evaluator observes the token instead. + var runTask = Task.Run(async () => + { + Volatile.Write(ref state.RunStartedTimestamp, Stopwatch.GetTimestamp()); + started.TrySetResult(); + var before = GC.GetAllocatedBytesForCurrentThread(); + try + { + // ScriptEvaluator.RunAsync executes the compiled delegate synchronously and returns a completed + // task, so the after-sample is taken on the same thread, right after the script returned. + var pending = evaluator.RunAsync(context, runToken); + slot.AllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - before; + return await pending.ConfigureAwait(false); + } + catch + { + slot.AllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - before; + throw; + } + }); + + // Release the slot when the run REALLY finishes — which for an abandoned run is long after this + // method has thrown ScriptTimeoutException. Registered before the wait so it fires regardless of + // which way the wait below comes out. A late-finishing orphan that breached the allocation budget + // still quarantines its key here. + _ = runTask.ContinueWith(t => + { + Volatile.Write(ref state.RunStartedTimestamp, 0); + Interlocked.Exchange(ref state.InFlight, 0); + Interlocked.Decrement(ref _inFlight); + runCts.Dispose(); + if (t.IsFaulted) _ = t.Exception; // observed below by the awaiter when it is still listening + if (t.IsCompletedSuccessfully && BreachedBudget(slot.AllocatedBytes)) + Quarantine(key, state, AllocationReason(slot.AllocatedBytes), "allocation"); + }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + + // The budget clock starts when the pool starts the run, not when we queued it (P5): pool starvation + // is a host problem and must not be reported as "script timed out". + await started.Task.WaitAsync(ct).ConfigureAwait(false); + + TResult result; + try + { + result = await runTask.WaitAsync(timeout, ct).ConfigureAwait(false); + } + catch (TimeoutException) + { + // The caller's cancel wins even when it lands a few µs after the timeout fired. + if (ct.IsCancellationRequested) throw new OperationCanceledException(ct); + runCts.Cancel(); + if (NoteTimeout(key, state, timeout)) + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + throw new ScriptTimeoutException(timeout); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception) + { + // The script returned control (by throwing) — that is a completion for quarantine purposes. + Interlocked.Exchange(ref state.ConsecutiveTimeouts, 0); + throw; + } + + Interlocked.Exchange(ref state.ConsecutiveTimeouts, 0); + if (BreachedBudget(slot.AllocatedBytes)) + { + // The continuation above has already (or will have) quarantined the key; make the reason visible + // to THIS caller too, so the breaching run itself publishes Bad rather than a suspect value. + Quarantine(key, state, AllocationReason(slot.AllocatedBytes), "allocation"); + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + } + return result; + } + + /// Resolve the key's state, rolling it over when the source identity changed, and refuse a quarantined key. + private KeyState Admit(string key, string? sourceIdentity) + { + while (true) + { + var state = _keys.GetOrAdd(key, static (_, src) => new KeyState(src), sourceIdentity); + if (sourceIdentity is not null && state.SourceIdentity is not null + && !string.Equals(state.SourceIdentity, sourceIdentity, StringComparison.Ordinal)) + { + // The script changed under this key: start from a clean slate. The old state's orphan (if any) + // releases the GLOBAL in-flight count through its own continuation; nothing else references it. + _keys.TryUpdate(key, new KeyState(sourceIdentity), state); + continue; + } + if (sourceIdentity is not null && state.SourceIdentity is null) + { + state.SourceIdentity = sourceIdentity; + } + if (state.QuarantineReason is { } reason) + throw new ScriptQuarantinedException(key, reason); + return state; + } + } + + /// Record a timeout strike; returns true when it tipped the key into quarantine. A completed run + /// (value or throw) resets the streak; a refusal neither counts nor resets. + private bool NoteTimeout(string key, KeyState state, TimeSpan timeout) + { + var count = Interlocked.Increment(ref state.ConsecutiveTimeouts); + if (count < _quarantineThreshold) return false; + Quarantine(key, state, + $"{count} consecutive evaluation(s) timed out (budget {timeout.TotalMilliseconds:F0} ms); " + + "the script is not evaluated again until its source changes or it is redeployed", + "timeouts"); + return true; + } + + private bool BreachedBudget(long allocatedBytes) => + _maxAllocBytesPerRun > 0 && allocatedBytes > _maxAllocBytesPerRun; + + private string AllocationReason(long allocatedBytes) => + $"one evaluation allocated {allocatedBytes / (1024.0 * 1024.0):F1} MB, over the " + + $"{_maxAllocBytesPerRun / (1024.0 * 1024.0):F0} MB budget (Scripting:MaxAllocBytesPerRun); " + + "the script is not evaluated again until its source changes or it is redeployed"; + + private void Quarantine(string key, KeyState state, string why, string metricReason) + { + var reason = QuarantinedReasonPrefix + why; + // First writer wins: a late orphan and the awaiter can both reach here for the same run. + if (Interlocked.CompareExchange(ref state.QuarantineReason, reason, null) is not null) return; + + Interlocked.Increment(ref _quarantined); + QuarantinedCounter.Add(1, + new KeyValuePair("script", key), + new KeyValuePair("reason", metricReason)); + _logger.Error("Script {ScriptKey} quarantined: {Reason}", key, why); + try { Quarantined?.Invoke(this, new ScriptQuarantinedEventArgs(key, reason)); } + catch (Exception ex) + { + _logger.Warning(ex, "ScriptRunGuard Quarantined subscriber threw for {ScriptKey}", key); + } + } + + private sealed class KeyState(string? sourceIdentity) + { + public string? SourceIdentity = sourceIdentity; + public int InFlight; + public int ConsecutiveTimeouts; + /// when the pool started the in-flight run; 0 when idle. + public long RunStartedTimestamp; + public string? QuarantineReason; + } + + private sealed class RunSlot + { + public long AllocatedBytes; + } +} + +/// Carried by . +/// The quarantined script instance key. +/// Why, prefixed . +public sealed record ScriptQuarantinedEventArgs(string Key, string Reason); + +/// +/// Thrown by when the script's previous run is +/// still executing (an abandoned, timed-out run that never returned). Nothing was dispatched; the seam +/// publishes Bad with . +/// +public sealed class ScriptRunRefusedException : Exception +{ + /// The script instance key. + public string Key { get; } + + /// Initializes the exception for . + /// The script instance key. + public ScriptRunRefusedException(string key) : base(ScriptRunGuard.PreviousRunStillRunningReason) + { + Key = key; + } +} + +/// +/// Thrown by for a quarantined script: repeated +/// timeouts, a stuck orphan or an allocation-budget breach. Nothing is dispatched while quarantined; the seam +/// publishes Bad carrying the reason, which an operator clears by fixing the script (source change) or +/// redeploying. +/// +public sealed class ScriptQuarantinedException : Exception +{ + /// The script instance key. + public string Key { get; } + + /// Initializes the exception with the quarantine reason (already prefixed). + /// The script instance key. + /// The quarantine reason, prefixed . + public ScriptQuarantinedException(string key, string reason) : base(reason) + { + Key = key; + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs new file mode 100644 index 00000000..296d862c --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs @@ -0,0 +1,56 @@ +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Node-wide script runtime-safety knobs (the Scripting appsettings section), consumed by +/// . One instance is shared by every script seam on the node — the +/// virtual-tag evaluator, the Calculation evaluator and the scripted-alarm engine — so a single setting +/// bounds every authored script the same way. +/// +/// +/// +/// The in-process sandbox cannot pre-empt a script: a while(true) is abandoned on its pool +/// thread at the timeout, and an allocation is observed only after the run returns. These knobs bound +/// the damage — one orphan per script, no further runs until the source changes — not the +/// first occurrence. Pre-emptive CPU/memory budgeting needs an out-of-process script runner. +/// +/// +public sealed class ScriptingOptions +{ + /// The appsettings section this binds from. + public const string SectionName = "Scripting"; + + /// Default : 64 MB. + public const long DefaultMaxAllocBytesPerRun = 64L * 1024 * 1024; + + /// Default : 3. + public const int DefaultQuarantineAfterConsecutiveTimeouts = 3; + + /// Default : 10. + public const double DefaultStuckOrphanQuarantineFactor = 10; + + /// + /// Managed bytes one script run may allocate on its own thread (sampled with + /// around the run) before the script is + /// quarantined. Zero or negative disables the budget. Default 64 MB. + /// + public long MaxAllocBytesPerRun { get; set; } = DefaultMaxAllocBytesPerRun; + + /// + /// Consecutive timeouts after which the script is quarantined: no further runs are dispatched, every + /// evaluation reports Bad, and the quarantine lifts only on a source change, a redeploy or an explicit + /// reset. Only a timeout is a strike — a run refused because the previous one is still executing is + /// reported Bad and counted, but neither adds to nor resets the streak; a completed run (value or + /// throw) resets it. Default 3; values below 1 are treated as 1. + /// + public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts; + + /// + /// A refusal quarantines the script immediately when the abandoned run it was refused behind has been + /// executing for longer than this many times the script's timeout (20 s at the 2 s default). This is + /// what quarantines a permanently stuck script (while(true)): it times out once and is then + /// only ever refused, so the timeout streak alone would never reach + /// . A young orphan (a short overrun) is still just a + /// refusal. Zero or negative disables the rule. Default 10. + /// + public double StuckOrphanQuarantineFactor { get; set; } = DefaultStuckOrphanQuarantineFactor; +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs index 4adeccab..7a860436 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs @@ -19,10 +19,12 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// task continues running on a thread-pool thread until the Roslyn runtime /// returns. In the CPU-bound-infinite-loop case that's effectively "leaked" — /// the thread is tied up until the runtime decides to return, which it may -/// never do. Phase 7 plan Stream A.4 accepts this as a known trade-off; tighter -/// CPU budgeting would require an out-of-process script runner, which is a v3 -/// concern. In practice, the timeout + structured warning log surfaces the -/// offending script so the operator can fix it; the orphan thread is rare. +/// never do. This wrapper has no memory of that orphan, so re-running the same +/// script through it on every trigger leaks one more thread per trigger — which +/// is why the production seams (virtual tags, Calculation, scripted alarms) run +/// through instead: same timeout, plus a per-script +/// "previous run still executing" refusal and a quarantine. This class remains +/// for callers that evaluate once (tests, the authoring preview). /// /// /// Caller-supplied is honored — if the caller diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs index d556ccf6..9f8ef41b 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs @@ -92,7 +92,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // the timer ticks — the mutual exclusion _lock used to provide while it was held across the run, minus // the blocking. Taken ONLY by evaluation workers, so no host thread (mux callback, ReadAsync, // reinit) can ever wait on a script. Lock order is always _evalGate → _lock, never the reverse. - private readonly object _evalGate = new(); + // A SemaphoreSlim rather than a monitor because the holder AWAITS the script run (the pump is async — + // it no longer parks a second pool thread behind the one the script runs on, and the run-guard's + // budget clock starts only once the pool has picked the script up, so pool starvation is never + // reported as a script timeout). + private readonly SemaphoreSlim _evalGate = new(1, 1); + + // Cancels an awaited script run on Dispose (the only hard teardown — ShutdownAsync leaves an in-flight + // run to its RunTimeout, exactly as before, since the pump drops its result once _running is false). + private readonly CancellationTokenSource _evalCts = new(); // Change-triggered evaluations queued by OnDependencyValue and drained, in arrival order, by a single // pump task. Both fields are guarded by _lock; _pumpScheduled stays true until the pump observes an @@ -103,7 +111,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba private CalculationTimerScheduler? _timers; private volatile bool _running; private DateTime? _lastComputeUtc; - private bool _disposed; + private volatile bool _disposed; /// Occurs when a calc tag's computed value changes (or transitions Good↔Bad). public event EventHandler? OnDataChange; @@ -129,7 +137,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // _logger (ILogger) implements the non-generic ILogger, so it is passed straight // through — the evaluator's host-side diagnostics (compile errors, sandbox violations, execution // throws) land on the SAME logger the driver itself was constructed with, not a separate one. - _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout); + _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout, _options.Scripting); RebuildTagTable(_options); } @@ -225,6 +233,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba { RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [], RunTimeout = _options.RunTimeout, + Scripting = _options.Scripting, }; } } @@ -235,7 +244,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba _driverInstanceId); } } - return new CalculationDriverOptions { RawTags = _options.RawTags, RunTimeout = _options.RunTimeout }; + return new CalculationDriverOptions + { + RawTags = _options.RawTags, RunTimeout = _options.RunTimeout, Scripting = _options.Scripting, + }; } /// Re-bind the tag table from — shared by @@ -326,6 +338,11 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba StopTimers(); // The pump exits at its next iteration; an evaluation already in flight is bounded by RunTimeout. lock (_lock) _pending.Clear(); + // R07:L2 — nothing in the runtime disposes a driver, and every config change is a respawn, so + // the compiled scripts' collectible ALCs survived each generation until GC got round to the whole + // driver graph. Drop them here (idempotent; a later InitializeAsync recompiles on first use). An + // in-flight run that races this is already handled by the evaluator's retry-once path. + _evaluator.ClearCompiledScripts(); return Task.CompletedTask; } @@ -351,11 +368,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba /// public void OnDependencyValue(string rawPath, object? value, uint statusCode, DateTime timestampUtc) { - if (!_running) return; lock (_lock) { + // Recorded REGARDLESS of _running (R07:H6): the mux adapter registers interest at construction, + // so inputs — the mux's retained-value replay in particular — can land before InitializeAsync. + // Dropping them left every calc tag waiting for each input to change again; keeping them means + // the first trigger after init evaluates from current inputs. Only the EVALUATION is gated. _depValues[rawPath] = new DataValueSnapshot(value, statusCode, timestampUtc, timestampUtc); _arrivedDeps.Add(rawPath); + if (!_running) return; if (!_changeDependents.TryGetValue(rawPath, out var dependents) || dependents.Count == 0) return; // QUEUE each change-triggered dependent — never evaluate here. This @@ -491,13 +512,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba { if (_pumpScheduled) return; _pumpScheduled = true; - _ = Task.Run(RunEvaluationPump); + _ = Task.Run(RunEvaluationPumpAsync); } - /// Drain the queued change-triggered evaluations, in arrival order, on a pool thread. At most - /// one pump runs at a time, so per-tag evaluation ordering is preserved exactly as it was when the - /// change path evaluated inline under the state lock. - private void RunEvaluationPump() + /// Drain the queued change-triggered evaluations, in arrival order. At most one pump runs at a + /// time, so per-tag evaluation ordering is preserved exactly as it was when the change path evaluated + /// inline under the state lock. Async: the pump AWAITS each script run instead of parking a pool thread + /// behind the one the script executes on (P5 — sync-over-async with a nested Task.Run manufactured + /// false timeouts under pool starvation). + private async Task RunEvaluationPumpAsync() { while (true) { @@ -515,39 +538,69 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // _evalGate (never _lock) is held across the run, so the pump and the timer ticks still // evaluate one at a time while no host thread can block behind a script. - lock (_evalGate) + if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed + try { - try - { - RunEvaluation(next); - } - catch (Exception ex) - { - // The evaluator never throws for script faults, so this is a defect path only — it - // must not kill the pump and strand every queued evaluation behind it. - _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed", - _driverInstanceId, next.Definition.RawPath); - } + await RunEvaluationAsync(next).ConfigureAwait(false); + } + catch (Exception ex) + { + // The evaluator never throws for script faults, so this is a defect path only — it + // must not kill the pump and strand every queued evaluation behind it. + _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed", + _driverInstanceId, next.Definition.RawPath); + } + finally + { + _evalGate.Release(); } } } - /// Timer-trigger entry point — prepares and evaluates a single calc tag on the scheduler's - /// pool thread (the scheduler's per-group in-flight guard is what bounds outstanding timer work). - private void EvaluateTagFromTimer(string rawPath) + /// Timer-trigger entry point — prepares and evaluates a single calc tag (the scheduler's + /// per-group in-flight guard is what bounds outstanding timer work; it stays set until this task + /// completes). + private async Task EvaluateTagFromTimerAsync(string rawPath) { if (!_running) return; PendingEvaluation? prepared; lock (_lock) prepared = PrepareEvaluation(rawPath, DateTime.UtcNow); if (prepared is null) return; - lock (_evalGate) RunEvaluation(prepared); + if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed + try + { + await RunEvaluationAsync(prepared).ConfigureAwait(false); + } + finally + { + _evalGate.Release(); + } + } + + /// Acquire ; false once has disposed it (a pump + /// iteration or timer tick that raced the teardown simply stops). + private async Task TryEnterEvalGateAsync() + { + if (_disposed) return false; + try + { + await _evalGate.WaitAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + return false; + } + if (!_disposed) return true; + // Dispose landed while we waited (it gave up the gate without disposing it): back out. + _evalGate.Release(); + return false; } /// Run one prepared evaluation and publish the outcome. MUST be called under /// and NOT under : the script run is the whole point of the /// split, and the script log + fan-out both invoke subscriber code. /// The prepared evaluation. - private void RunEvaluation(PendingEvaluation pending) + private async Task RunEvaluationAsync(PendingEvaluation pending) { var def = pending.Definition; @@ -572,9 +625,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // Snapshots, not bare values: the script's ctx.GetTag(path).StatusCode / .SourceTimestampUtc are // then the input's real ones rather than a fabricated Good stamped at evaluation time. - var result = _evaluator.Evaluate(def.RawPath, def.ScriptSource, pending.Dependencies); + var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token) + .ConfigureAwait(false); - string? failureReason = null; + string? failureToLog = null; (object? Value, uint StatusCode)? toPublish; lock (_lock) { @@ -584,16 +638,24 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba if (!result.Success) { - failureReason = result.Reason ?? "evaluator failure"; + var reason = result.Reason ?? "evaluator failure"; toPublish = MarkEvaluationFailed(st, pending.TimestampUtc); + // The script-log entry fires on the Good→Bad TRANSITION, or when the failure changes + // (R07:P4) — not on every trigger of a tag that is already Bad for the same reason, which + // flooded the script log at the input's cadence. The Bad publish was already once per + // transition; the log now matches it. + if (toPublish is not null || !string.Equals(st.LastFailureReason, reason, StringComparison.Ordinal)) + failureToLog = reason; + st.LastFailureReason = reason; } else { + st.LastFailureReason = null; toPublish = MarkEvaluationSucceeded(st, result.Value, pending.WorstInputStatus, pending.TimestampUtc); } } - if (failureReason is not null) PublishScriptLog(def, failureReason); + if (failureToLog is not null) PublishScriptLog(def, failureToLog); if (toPublish is { } p) Publish(def.RawPath, p.Value, p.StatusCode, pending.TimestampUtc); } @@ -612,7 +674,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba private (object? Value, uint StatusCode)? MarkEvaluationSucceeded( TagState st, object? value, uint statusCode, DateTime timestampUtc) { - if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && Equals(st.LastValue, value)) + if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && ValuesEqual(st.LastValue, value)) return null; st.HasValue = true; @@ -624,6 +686,17 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba return (value, statusCode); } + /// Value equality for the dedup: structural for arrays (R07:H8 — object.Equals is reference + /// equality for a double[], so an array-returning timer script republished every tick through + /// the dual-namespace fan-out and the historian), ordinary + /// otherwise. + private static bool ValuesEqual(object? a, object? b) + { + if (a is Array x && b is Array y) + return System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(x, y); + return Equals(a, b); + } + /// Record that this tag's INPUTS are Bad, so no evaluation ran. Publishes the /// worst input's status — carrying the last known value — once per change of that status: a /// repeat of the same Bad is silent, while BadNoCommunication arriving over an earlier @@ -688,7 +761,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba { var timed = _tagsByRawPath.Values.Where(t => t.TimerInterval is not null).ToArray(); if (timed.Length == 0) return; - _timers = new CalculationTimerScheduler(EvaluateTagFromTimer, _logger); + _timers = new CalculationTimerScheduler(EvaluateTagFromTimerAsync, _logger); _timers.Start(timed); } @@ -701,6 +774,12 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba /// Test/diagnostic accessor: number of timer ticks skipped for in-flight groups. internal long TimerSkippedTickCount => _timers?.SkippedTickCount ?? 0; + /// Test/diagnostic accessor: sources currently remembered as failing to compile. + internal int CompileFailureCountForTest => _evaluator.CompileFailureCount; + + /// Test/diagnostic accessor: compiled scripts currently held by the evaluator's cache. + internal int CompiledScriptCountForTest => _evaluator.CompiledScriptCount; + /// Test/diagnostic accessor: when no change-triggered evaluation is /// queued or in flight. A test that used to assert straight after — /// which evaluated inline — waits on this instead, so the assertion still runs against a settled @@ -723,9 +802,34 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba _running = false; StopTimers(); lock (_lock) _pending.Clear(); + // Cancel an awaited run first (the guard lets the caller's cancel win over the timeout), then drop + // the compiled scripts. + try { _evalCts.Cancel(); } catch (ObjectDisposedException) { } _evaluator.Dispose(); + + // Then take the gate: once Dispose holds it no evaluation is mid-run (the token — read only under + // the gate — has been observed), so the gate and the CTS can be disposed. A late pump iteration or + // timer tick sees _disposed / an ObjectDisposedException in TryEnterEvalGateAsync and stops. Bounded: + // a run that ignores the cancel cannot hold Dispose; in that case both are left to the GC. + if (_evalGate.Wait(DisposeGateTimeout)) + { + _evalCts.Dispose(); + _evalGate.Dispose(); + _evalResourcesDisposed = true; + } + else + { + _logger.LogWarning("Calculation {Driver}: an evaluation did not release the gate within {Timeout} of Dispose; leaving it to the GC", + _driverInstanceId, DisposeGateTimeout); + } } + private static readonly TimeSpan DisposeGateTimeout = TimeSpan.FromSeconds(2); + private volatile bool _evalResourcesDisposed; + + /// Test hook — whether disposed the evaluation gate and its CTS. + internal bool EvalResourcesDisposedForTest => _evalResourcesDisposed; + /// Per-tag published state (guarded by ). private sealed class TagState { @@ -741,6 +845,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba /// now carries the failing INPUT's sub-code rather than a fixed BadInternalError. public uint LastPublishedStatus; public DateTime LastTimestampUtc; + + /// The reason of the last failed evaluation (null after a success / inputs-Bad hold) — the + /// gate that keeps the script-log entry to the transition and to a CHANGE of reason. + public string? LastFailureReason; } private sealed record CalculationSubscriptionHandle(string DiagnosticId) : ISubscriptionHandle; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs index ab10f06d..1a589d65 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs @@ -71,11 +71,22 @@ public static class CalculationDriverFactoryExtensions ?? throw new InvalidOperationException( $"Calculation driver config for '{driverInstanceId}' deserialised to null"); + // Run-guard knobs are optional and default to the node-wide ScriptingOptions defaults; only an + // explicitly authored key overrides one (so a config carrying neither binds Scripting = null). + ScriptingOptions? scripting = null; + if (dto.MaxAllocBytesPerRun is not null || dto.QuarantineAfterConsecutiveTimeouts is not null) + { + scripting = new ScriptingOptions(); + if (dto.MaxAllocBytesPerRun is { } alloc) scripting.MaxAllocBytesPerRun = alloc; + if (dto.QuarantineAfterConsecutiveTimeouts is { } n) scripting.QuarantineAfterConsecutiveTimeouts = n; + } + var options = new CalculationDriverOptions { RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [], RunTimeout = dto.RunTimeoutMs is { } ms && ms > 0 ? TimeSpan.FromMilliseconds(ms) : TimeSpan.FromSeconds(2), + Scripting = scripting, }; return new CalculationDriver( @@ -92,5 +103,11 @@ public static class CalculationDriverFactoryExtensions /// Optional per-script evaluation wall-clock budget override, in milliseconds (default 2000). public int? RunTimeoutMs { get; init; } + + /// Optional run-guard allocation budget per script run, in bytes (default 64 MB; ≤ 0 disables). + public long? MaxAllocBytesPerRun { get; init; } + + /// Optional run-guard quarantine threshold: consecutive timeouts (refusals do not count) before a tag's script stops being evaluated (default 3). + public int? QuarantineAfterConsecutiveTimeouts { get; init; } } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs index b6642c09..f00f50e4 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs @@ -1,20 +1,21 @@ using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; -/// -/// Bound options for a instance. The Calculation pseudo-driver has -/// no backend/endpoint config — its only input is the authored raw calc tags the deploy artifact -/// injects as the list (each carrying the tag's scriptId + resolved -/// scriptSource + trigger config inside its blob). -/// +/// Bound options for one instance. public sealed class CalculationDriverOptions { - /// The driver's authored raw calc tags (RawPath + TagConfig blob + WriteIdempotent), from - /// the deploy artifact. Each is mapped to a at construction. + /// The authored raw calc tags (RawPath + TagConfig JSON) this driver computes. public IReadOnlyList RawTags { get; init; } = Array.Empty(); - /// Per-script wall-clock evaluation budget handed to the . - /// Defaults to 2 seconds (parity with the VirtualTag evaluator). + /// Wall-clock budget per script run (driver-config runTimeoutMs); default 2 s. public TimeSpan RunTimeout { get; init; } = TimeSpan.FromSeconds(2); + + /// + /// Run-guard budget + quarantine threshold for this driver's scripts (driver-config + /// maxAllocBytesPerRun / quarantineAfterConsecutiveTimeouts). Null ⇒ the + /// defaults (64 MB, 3). + /// + public ScriptingOptions? Scripting { get; init; } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs index 249c6958..dd7085e3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using Microsoft.Extensions.Logging; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; @@ -37,8 +38,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// apply-boundary clear, which is what stops collectible AssemblyLoadContexts accreting across /// script edits. /// +/// +/// Run quarantine. Every compiled run goes through one keyed by +/// calc-tag RawPath with the script source as the identity: a run is refused while the tag's previous +/// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive +/// timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach — both surface as a +/// the driver maps to Bad. The driver's evaluation pump awaits rather than +/// blocking a pool thread on the sync , +/// and the guard starts the budget clock only once the pool has picked the script up, so pool starvation +/// is never reported as a script timeout. +/// /// -public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable +public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineOwner, IDisposable { // Source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and Clear()/Dispose() // that dispose each evaluator's collectible AssemblyLoadContext. ClearCompiledScripts() — driven by the @@ -47,6 +58,14 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable private readonly ILogger _logger; private readonly SerilogLogger _scriptRoot; private readonly TimeSpan _runTimeout; + private readonly ScriptRunGuard _guard; + + // Negative compile cache (R07:P4), keyed by source hash. A compile error is a property of the SOURCE, + // which never changes between triggers (it is fixed in the deploy artifact) — yet every trigger of a + // broken tag used to re-run a full Roslyn compile under the driver-wide evaluation gate, starving the + // healthy tags behind it, and log the same Warning each time. The failure is now remembered until the + // next ClearCompiledScripts (a redeploy), and the host Warning fires once per source. + private readonly ConcurrentDictionary _compileFailures = new(StringComparer.Ordinal); private bool _disposed; /// Initializes a new . @@ -56,16 +75,22 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable /// implements this — instead of needing a second, separately-obtained logger instance. /// Root script logger; user ctx.Logger.* output flows through this to the Script-log page. /// Wall-clock budget per script run; defaults to 2 seconds (parity with the VT evaluator). + /// Run-guard budget + quarantine threshold; null ⇒ defaults. public CalculationEvaluator( ILogger logger, ScriptRootLogger scriptRoot, - TimeSpan? runTimeout = null) + TimeSpan? runTimeout = null, + ScriptingOptions? scripting = null) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger; _runTimeout = runTimeout ?? TimeSpan.FromSeconds(2); + _guard = new ScriptRunGuard(scripting, _scriptRoot); } + /// The run guard every compiled evaluation goes through (diagnostics / tests). + public ScriptRunGuard RunGuard => _guard; + /// /// Evaluate for the calc tag against /// the snapshot in (RawPath → last value). Never throws — script @@ -112,8 +137,25 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable /// The C# script source producing the tag's value. /// Read-only snapshot of dependency values keyed by RawPath. /// The evaluation outcome. + /// Blocks the calling thread for the run. The driver's pump uses ; + /// this overload remains for callers with no async context — chiefly tests. public VirtualTagEvalResult Evaluate( string calcTagId, string scriptSource, IReadOnlyDictionary dependencies) + => EvaluateAsync(calcTagId, scriptSource, dependencies).GetAwaiter().GetResult(); + + /// + /// The async form of : + /// the compiled script runs on the pool under the and the caller awaits + /// it instead of blocking a second pool thread for the duration. Never throws for script faults. + /// + /// Identity of the calc tag (bound to the script log). + /// The C# script source producing the tag's value. + /// Read-only snapshot of dependency values keyed by RawPath. + /// Caller cancellation (driver shutdown); wins over the run timeout. + /// The evaluation outcome. + public async Task EvaluateAsync( + string calcTagId, string scriptSource, IReadOnlyDictionary dependencies, + CancellationToken ct = default) { if (_disposed) return VirtualTagEvalResult.Failure("evaluator disposed"); if (string.IsNullOrWhiteSpace(scriptSource)) return VirtualTagEvalResult.Failure("empty expression"); @@ -146,6 +188,11 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable calcTagId, path), logger: scriptLog); + // A source that already failed to compile fails the same way without touching Roslyn (R07:P4). + var sourceKey = HashSource(scriptSource); + if (_compileFailures.TryGetValue(sourceKey, out var knownFailure)) + return VirtualTagEvalResult.Failure(knownFailure); + // Fetch + run inside a bounded loop so an ObjectDisposedException caused by a concurrent // ClearCompiledScripts (which disposed the evaluator between GetOrCompile and the run's // disposed guard) re-fetches and retries exactly once. The loop runs at most twice. @@ -158,13 +205,11 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable } catch (CompilationErrorException ex) { - _logger.LogWarning(ex, "Calculation {Id}: Roslyn compile failed", calcTagId); - return VirtualTagEvalResult.Failure($"compile error: {ex.Message}"); + return RememberCompileFailure(sourceKey, calcTagId, $"compile error: {ex.Message}", ex, "Roslyn compile failed"); } catch (ScriptSandboxViolationException ex) { - _logger.LogWarning(ex, "Calculation {Id}: sandbox violation", calcTagId); - return VirtualTagEvalResult.Failure($"sandbox violation: {ex.Message}"); + return RememberCompileFailure(sourceKey, calcTagId, $"sandbox violation: {ex.Message}", ex, "sandbox violation"); } catch (ObjectDisposedException) { @@ -180,11 +225,12 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable try { - // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't - // interrupt a CPU-bound/infinite-loop Roslyn script, so the timed wrapper is what bounds a - // runaway script to the wall-clock budget instead of wedging the driver's dispatch. - var timed = new TimedScriptEvaluator(evaluator, _runTimeout); - var raw = timed.RunAsync(context).GetAwaiter().GetResult(); + // Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by RawPath): a raw + // CancellationToken can't interrupt a CPU-bound/infinite-loop Roslyn script, so the wall-clock + // budget is what bounds a runaway script — and the guard is what keeps the next trigger from + // stacking another orphan behind it (one per tag, then quarantine). + var raw = await _guard.RunAsync(calcTagId, scriptSource, evaluator, context, _runTimeout, ct) + .ConfigureAwait(false); return VirtualTagEvalResult.Ok(raw); } catch (ScriptTimeoutException) @@ -192,6 +238,18 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable return VirtualTagEvalResult.Failure( $"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s"); } + catch (ScriptRunRefusedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } + catch (ScriptQuarantinedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + return VirtualTagEvalResult.Failure("evaluation cancelled (driver shutting down)"); + } catch (ObjectDisposedException) when (!_disposed && attempt == 0) { // A concurrent ClearCompiledScripts disposed the evaluator between our GetOrCompile and the @@ -211,8 +269,39 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable } } + /// Record a compile failure for and log it ONCE — the next trigger + /// of the same source returns the remembered reason without a Roslyn round-trip or a second Warning. + private VirtualTagEvalResult RememberCompileFailure(string sourceKey, string calcTagId, string reason, Exception ex, string what) + { + if (_compileFailures.TryAdd(sourceKey, reason)) + _logger.LogWarning(ex, "Calculation {Id}: {What} (remembered until the next redeploy)", calcTagId, what); + return VirtualTagEvalResult.Failure(reason); + } + + private static string HashSource(string source) + => Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(source))); + + /// Test/diagnostic: the number of sources currently remembered as failing to compile. + public int CompileFailureCount => _compileFailures.Count; + + /// Test/diagnostic: the number of compiled scripts currently cached. + public int CompiledScriptCount => _cache.Count; + /// - public void ClearCompiledScripts() => _cache.Clear(); + public void ClearCompiledScripts() + { + _cache.Clear(); + _compileFailures.Clear(); + // The compile-cache clear is this evaluator's deploy boundary — a redeploy lifts every quarantine + // (an edited source already lifts its own through the guard's source identity). + _guard.ResetAll(); + } + + /// + public void ResetQuarantines() => _guard.ResetAll(); + + /// + public IReadOnlyCollection QuarantinedScripts => _guard.QuarantinedKeys; /// Disposes the evaluator, draining + disposing every cached compile's collectible ALC. public void Dispose() @@ -220,5 +309,6 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable if (_disposed) return; _disposed = true; _cache.Dispose(); + _compileFailures.Clear(); } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs index ef74da0e..694f7d8a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs @@ -7,11 +7,13 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// per distinct interval group (so the wire count stays low /// regardless of tag count) with a per-group in-flight flag that skips a tick when the prior tick for /// the same group is still running. Each tick invokes the driver's own evaluate -/// callback per tag in the group. +/// callback per tag in the group and AWAITS it — the in-flight flag is held across the awaited script +/// run, not just across the synchronous part of the callback, so a slow script still bounds its group +/// to one outstanding tick. /// internal sealed class CalculationTimerScheduler : IDisposable { - private readonly Action _evaluate; + private readonly Func _evaluate; private readonly ILogger _logger; private readonly List _timers = []; private readonly List _groups = []; @@ -20,9 +22,9 @@ internal sealed class CalculationTimerScheduler : IDisposable private bool _disposed; /// Initializes a new instance of the class. - /// Callback invoked with a calc tag's RawPath when its timer group ticks. + /// Callback invoked with a calc tag's RawPath when its timer group ticks; awaited. /// Logger for diagnostics. - public CalculationTimerScheduler(Action evaluate, ILogger logger) + public CalculationTimerScheduler(Func evaluate, ILogger logger) { _evaluate = evaluate ?? throw new ArgumentNullException(nameof(evaluate)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -69,12 +71,20 @@ internal sealed class CalculationTimerScheduler : IDisposable return; } + // The Timer callback is void; the tick itself is async so the awaited script runs do not pin this + // pool thread. TickAsync owns the in-flight flag and swallows everything — nothing can escape to the + // Timer (which would be an unobserved fault on a timer thread). + _ = TickAsync(ctx); + } + + private async Task TickAsync(TickGroup ctx) + { try { foreach (var path in ctx.Paths) { if (_cts.IsCancellationRequested) return; - try { _evaluate(path); } + try { await _evaluate(path).ConfigureAwait(false); } catch (Exception ex) { _logger.LogError(ex, "CalculationTimerScheduler evaluate failed for {Path}", path); } } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs index 6242351c..6e3bc5f8 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GalaxyDiscoverer.cs @@ -61,8 +61,12 @@ public sealed class GalaxyDiscoverer /// /// The address space builder to populate with discovery results. /// The cancellation token for the operation. - /// A task that represents the asynchronous operation. - public async Task DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken) + /// + /// The hierarchy this pass fetched, so a caller that needs the object set for something + /// else (the driver selects its $WinPlatform / $AppEngine probe targets from + /// it) reuses the one sweep instead of asking the gateway for the full galaxy twice. + /// + public async Task> DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(builder); var objects = await _source.GetHierarchyAsync(cancellationToken).ConfigureAwait(false); @@ -132,6 +136,8 @@ public sealed class GalaxyDiscoverer } } } + + return objects; } // Resolve (and memoise) the folder for a gobject, recursively creating its parent's folder diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs index bafabb59..68899bb8 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyHierarchySource.cs @@ -5,22 +5,83 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; /// /// Default wrapping the gateway's -/// . Pages internally via the client's overload. +/// . Pages the DiscoverHierarchy RPC itself, +/// at the configured Repository.DiscoverPageSize: the client's typed +/// DiscoverHierarchyAsync overloads hard-code a 5000-row page +/// (DiscoverHierarchyOptions carries no page size), so the knob the AdminUI +/// authors and the factory parses used to reach nothing. The raw RPC takes the page +/// size on the wire (DiscoverHierarchyRequest.page_size), so it is honoured here. /// public sealed class GatewayGalaxyHierarchySource : IGalaxyHierarchySource { + /// The page size used when the configured one is not positive — the client library's own default. + public const int DefaultPageSize = 5000; + private readonly GalaxyRepositoryClient _client; + private readonly int _pageSize; /// /// Initializes a new instance of the class. /// /// The gateway's Galaxy repository client for discovering the object hierarchy. - public GatewayGalaxyHierarchySource(GalaxyRepositoryClient client) + /// + /// Objects per DiscoverHierarchy page (Repository.DiscoverPageSize). A value that is + /// not positive falls back to . + /// + public GatewayGalaxyHierarchySource(GalaxyRepositoryClient client, int pageSize = DefaultPageSize) { _client = client ?? throw new ArgumentNullException(nameof(client)); + _pageSize = EffectivePageSize(pageSize); } + /// The page size this source sends on every DiscoverHierarchy request. + public int PageSize => _pageSize; + + /// Clamp a configured page size to something the gateway accepts: non-positive → . + /// The configured value. + /// The page size to send. + public static int EffectivePageSize(int configured) => configured > 0 ? configured : DefaultPageSize; + /// public Task> GetHierarchyAsync(CancellationToken cancellationToken) - => _client.DiscoverHierarchyAsync(cancellationToken); + => PageAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken); + + /// + /// Walk every DiscoverHierarchy page at and concatenate the + /// objects. Mirrors the client library's own loop (including its repeated-page-token guard, + /// which turns a misbehaving gateway into an error rather than an endless walk), with the + /// RPC behind a delegate so the loop is unit-testable without a channel. + /// + /// The raw DiscoverHierarchy RPC. + /// Objects per page; sent on every request. + /// Cancellation token for the walk. + /// Every object across all pages, in gateway order. + internal static async Task> PageAsync( + Func> discover, + int pageSize, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(discover); + var effective = EffectivePageSize(pageSize); + + var objects = new List(); + var seenPageTokens = new HashSet(StringComparer.Ordinal); + var pageToken = string.Empty; + do + { + var request = new DiscoverHierarchyRequest { PageSize = effective, PageToken = pageToken }; + var reply = await discover(request, cancellationToken).ConfigureAwait(false); + + objects.AddRange(reply.Objects); + pageToken = reply.NextPageToken ?? string.Empty; + if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken)) + { + throw new InvalidOperationException( + $"Galaxy DiscoverHierarchy returned a repeated page token '{pageToken}' — refusing to loop."); + } + } + while (!string.IsNullOrWhiteSpace(pageToken)); + + return objects; + } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs new file mode 100644 index 00000000..dfed1659 --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs @@ -0,0 +1,88 @@ +using ZB.MOM.WW.MxGateway.Client; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; + +/// +/// Default : a platform-only DiscoverHierarchy sweep +/// over the gateway's — the request filters on +/// template_chain_contains ($WinPlatform, $AppEngine) and asks for no +/// attributes, so it is a few rows per galaxy rather than the full object + attribute dump the +/// address-space discovery pulls. Paged at the configured Repository.DiscoverPageSize. +/// +public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource +{ + /// Template-chain tokens that identify a platform / engine object. + public static readonly IReadOnlyList PlatformTemplates = ["$WinPlatform", "$AppEngine"]; + + private readonly GalaxyRepositoryClient _client; + private readonly int _pageSize; + + /// Initializes a new instance of the class. + /// The gateway's Galaxy repository client. + /// Objects per page (Repository.DiscoverPageSize); non-positive → the client default. + public GatewayGalaxyPlatformSource(GalaxyRepositoryClient client, int pageSize = GatewayGalaxyHierarchySource.DefaultPageSize) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + _pageSize = GatewayGalaxyHierarchySource.EffectivePageSize(pageSize); + } + + /// + public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + => QueryAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken); + + /// + /// The platform-only sweep behind a delegate so it is unit-testable without a channel. Every + /// request carries the template filter and include_attributes = false; the reply is + /// re-filtered on the returned template chain (a gateway that ignores the filter still + /// yields only platforms) and reduced to non-empty tag names. + /// + /// The raw DiscoverHierarchy RPC. + /// Objects per page. + /// Cancellation token for the walk. + /// Distinct platform tag names in gateway order. + internal static async Task> QueryAsync( + Func> discover, + int pageSize, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(discover); + var effective = GatewayGalaxyHierarchySource.EffectivePageSize(pageSize); + + var names = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var seenPageTokens = new HashSet(StringComparer.Ordinal); + var pageToken = string.Empty; + do + { + var request = new DiscoverHierarchyRequest + { + PageSize = effective, + PageToken = pageToken, + IncludeAttributes = false, + }; + request.TemplateChainContains.Add(PlatformTemplates); + + var reply = await discover(request, cancellationToken).ConfigureAwait(false); + foreach (var obj in reply.Objects) + { + if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue; + if (seen.Add(obj.TagName)) names.Add(obj.TagName); + } + + pageToken = reply.NextPageToken ?? string.Empty; + if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken)) + { + throw new InvalidOperationException( + $"Galaxy DiscoverHierarchy (platform query) returned a repeated page token '{pageToken}' — refusing to loop."); + } + } + while (!string.IsNullOrWhiteSpace(pageToken)); + + return names; + } + + private static bool IsPlatform(GalaxyObject obj) => + obj.TemplateChain.Count == 0 // a gateway that strips the chain already filtered server-side + || obj.TemplateChain.Any(t => PlatformTemplates.Contains(t, StringComparer.OrdinalIgnoreCase)); +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/IGalaxyPlatformSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/IGalaxyPlatformSource.cs new file mode 100644 index 00000000..6589c9aa --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/IGalaxyPlatformSource.cs @@ -0,0 +1,18 @@ +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; + +/// +/// Driver-side seam for the set of Galaxy platform / engine objects whose ScanState +/// the PerPlatformProbeWatcher advises. Production wraps the gateway's +/// DiscoverHierarchy RPC filtered to $WinPlatform / $AppEngine with no +/// attributes (); tests substitute a canned list. +/// The driver seeds the watcher from this at InitializeAsync and re-seeds on every +/// deploy-watcher rediscovery — the runtime never calls DiscoverAsync on a live driver, +/// so without this seed the probe watcher never had a platform set in production. +/// +public interface IGalaxyPlatformSource +{ + /// The tag_name of every deployed $WinPlatform / $AppEngine object. + /// Cancellation token for the query. + /// Platform tag names; empty when the galaxy has none. + Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken); +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs index 3216e136..d0fac54a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs @@ -146,9 +146,98 @@ public sealed class GalaxyDriver private HostConnectivityForwarder? _transportForwarder; private PerPlatformProbeWatcher? _probeWatcher; + // Injected deploy-watch seam (tests). Null in production, where StartDeployWatcherAsync builds a + // GatewayGalaxyDeployWatchSource over the owned GalaxyRepositoryClient. + private readonly IGalaxyDeployWatchSource? _deployWatchSource; + + // Platform set for the ScanState probe watcher. Injected seam (tests) or, on the production + // path, a platform-only DiscoverHierarchy query over the owned repository client. Seeded at + // InitializeAsync and re-seeded on every deploy-watcher rediscovery: the runtime never calls + // DiscoverAsync on a live driver (only the /raw picker's throwaway does), so SyncPlatformsAsync + // from DiscoverAsync alone left the production watcher with no platforms, ever. + private readonly IGalaxyPlatformSource? _platformSource; + private IGalaxyPlatformSource? _ownedPlatformSource; + // Probe-seed requests (init + every rediscovery) not yet served. The first request starts the + // background worker; later ones only bump the count, and the worker loops until it has served + // every request it saw — so a request that lands while a pass is mid-fetch (a deploy, or the + // next runtime's init after a teardown) still gets a pass that starts after it. + private int _reseedRequests; + private volatile string _seedReason = "initialize"; + + // Cancels the in-flight background seed of the CURRENT runtime. Created by InitializeAsync, + // cancelled (never disposed — it has no timer or wait handle, and the worker may still read its + // token) by the teardown. + private CancellationTokenSource? _seedCts; + + // Serialises "is this registration still for the live runtime?" against the teardown's registry + // Clear, so a subscribe / re-subscribe whose SubscribeBulk straddled a teardown either lands + // before the Clear (and is wiped by it) or sees the bumped generation and backs out. + private readonly Lock _registryGate = new(); + + // Bumped by every runtime teardown. Background work started under one runtime (the replay-gap + // re-snapshot, a rediscovery re-seed) re-checks it before touching the registry / watcher, so a + // late Rebind cannot resurrect handles into a registry the teardown already cleared. + private int _runtimeGeneration; + + /// + /// Upper bound on the gateway CloseSession round-trip during a teardown. The SDK's + /// default close runs under the 30 s call timeout, which against a blackholed gateway parks + /// ShutdownAsync — and the host's PostStop — for the whole of it. Internal so + /// tests can shrink it. + /// + internal TimeSpan SessionCloseDeadline { get; set; } = TimeSpan.FromSeconds(5); + + // True while the seams (_subscriber / _dataWriter / _hierarchySource / _alarmFeed / + // _alarmAcknowledger) were built by BuildProductionRuntimeAsync and are therefore ours to dispose + // and null on ShutdownAsync — the host's comms-loss recovery is ShutdownAsync → InitializeAsync on + // this SAME object, and InitializeAsync rebuilds only when the seams are null. Injected seams + // belong to the injector: ShutdownAsync stops every loop it started on top of them but leaves the + // seams in place, so a seam-driven test can cycle the lifecycle the way the watchdog does. + private bool _ownsRuntime; + + // Serialises InitializeAsync / ShutdownAsync / DisposeAsync so a late watchdog re-init can never + // interleave with a teardown and resurrect half of the runtime it is tearing down. + private readonly SemaphoreSlim _lifecycleGate = new(1, 1); + + // Set for the duration of a runtime teardown so the pump's stream-fault / replay-gap callbacks and + // the supervisor's replay cannot start a fresh pump underneath it. + private volatile bool _runtimeStopping; + private DriverHealth _health = new(DriverState.Unknown, null, null); private bool _disposed; + /// + /// Test seam — forwarded to on the + /// session creates, so the owned-runtime lifecycle + /// (client + session construction, teardown, rebuild) can be exercised without a gateway. The + /// gRPC channels it builds are real but lazy: nothing dials until a call is made. + /// + internal Func? SessionOpenOverrideForTests { get; set; } + + /// Test seam: the owned gateway client, null when no production runtime is live. + internal MxGatewayClient? OwnedMxClientForTests => _ownedMxClient; + + /// Test seam: the owned MX session, null when no production runtime is live. + internal GalaxyMxSession? OwnedMxSessionForTests => _ownedMxSession; + + /// Test seam: the owned repository client, null until the deploy watcher or discovery built it. + internal GalaxyRepositoryClient? OwnedRepositoryClientForTests => _ownedRepositoryClient; + + /// Test seam: the live reconnect supervisor, null outside a production runtime. + internal ReconnectSupervisor? SupervisorForTests => _supervisor; + + /// Test seam: the live per-platform probe watcher, null until InitializeAsync. + internal PerPlatformProbeWatcher? ProbeWatcherForTests => _probeWatcher; + + /// Test seam: the live deploy watcher, null until InitializeAsync starts one. + internal DeployWatcher? DeployWatcherForTests => _deployWatcher; + + /// Test seam: true while an is running. + internal bool HasEventPumpForTests { get { lock (_pumpLock) return _eventPump is not null; } } + + /// Test seam: the number of driver subscriptions the registry still tracks. + internal int TrackedSubscriptionCountForTests => _subscriptions.TrackedSubscriptionCount; + /// /// Server-pushed data-change notification. Fires from the /// 's background loop; handlers should be cheap (or queue @@ -202,6 +291,15 @@ public sealed class GalaxyDriver /// null-object resolver (returns null for every name) so seam-injecting tests that /// don't exercise a secret: ref need not supply one. /// + /// + /// Optional deploy-event stream seam. When supplied, starts the + /// over it (production builds one over the owned repository client). + /// + /// + /// Optional platform-set seam for the ScanState probe watcher. When supplied, + /// seeds the watcher from it and every rediscovery re-seeds + /// (production builds a over the owned repository client). + /// internal GalaxyDriver( string driverInstanceId, GalaxyDriverOptions options, @@ -212,8 +310,12 @@ public sealed class GalaxyDriver IGalaxyAlarmAcknowledger? alarmAcknowledger = null, IGalaxyAlarmFeed? alarmFeed = null, ILogger? logger = null, - ISecretResolver? secretResolver = null) + ISecretResolver? secretResolver = null, + IGalaxyDeployWatchSource? deployWatchSource = null, + IGalaxyPlatformSource? platformSource = null) { + _deployWatchSource = deployWatchSource; + _platformSource = platformSource; _driverInstanceId = !string.IsNullOrWhiteSpace(driverInstanceId) ? driverInstanceId : throw new ArgumentException("Driver instance id required.", nameof(driverInstanceId)); @@ -359,28 +461,195 @@ public sealed class GalaxyDriver ObjectDisposedException.ThrowIf(_disposed, this); if (HasConfigBody(driverConfigJson)) RefuseNonEquivalentConfig(driverConfigJson); - // Tests inject seams via the internal ctor; production InitializeAsync builds - // the gateway client + session + per-capability runtime components from - // GalaxyDriverOptions. When seams are pre-injected we leave them alone (the - // test exercises the wired surface without a real gw round-trip). - if (_subscriber is null && _dataWriter is null && _hierarchySource is null) + await _lifecycleGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try { - await BuildProductionRuntimeAsync(cancellationToken).ConfigureAwait(false); + ObjectDisposedException.ThrowIf(_disposed, this); + + // Tests inject seams via the internal ctor; production InitializeAsync builds + // the gateway client + session + per-capability runtime components from + // GalaxyDriverOptions. When seams are pre-injected we leave them alone (the + // test exercises the wired surface without a real gw round-trip). After a + // ShutdownAsync the production seams are null again, so the watchdog's + // Shutdown → Initialize genuinely rebuilds the runtime on this same object. + if (_subscriber is null && _dataWriter is null && _hierarchySource is null) + { + await BuildProductionRuntimeAsync(cancellationToken).ConfigureAwait(false); + } + else + { + _logger.LogDebug( + "GalaxyDriver {InstanceId} initializing with pre-injected seams — production runtime build skipped", + _driverInstanceId); + // The per-platform probe watcher needs only a subscriber, so it runs on the seam path + // too — that is what lets the ScanState wiring be exercised without a gateway. + if (_subscriber is not null) _probeWatcher ??= BuildProbeWatcher(_subscriber); + } + + await StartDeployWatcherAsync(cancellationToken).ConfigureAwait(false); + // The platform seed is a gateway DiscoverHierarchy round-trip, which can spend the whole of + // a cold gateway cache's first-load wait — longer than Galaxy's Tier-A Initialize budget. It + // runs in the background: InitializeAsync neither waits on it nor fails because of it. + await PrepareProbeSeedAsync(cancellationToken).ConfigureAwait(false); + RequestProbeSeed("initialize"); + _logger.LogInformation( + "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}", + _driverInstanceId, _options.Gateway.Endpoint, _options.MxAccess.ClientName); + _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null); } - else + finally { - _logger.LogDebug( - "GalaxyDriver {InstanceId} initializing with pre-injected seams — production runtime build skipped", + _lifecycleGate.Release(); + } + } + + /// + /// Under the lifecycle gate, before the first seed request of a runtime: resolve the platform + /// source (production builds it over the owned repository client — no network here) and arm + /// the runtime's seed cancellation. Resolving here, not in the worker, keeps the background + /// pass from creating a repository client after a teardown released it. A failure is a + /// Warning: the value path does not depend on the probe set. + /// + private async Task PrepareProbeSeedAsync(CancellationToken cancellationToken) + { + if (_probeWatcher is null) return; + _seedCts ??= new CancellationTokenSource(); + if (_platformSource is not null || !_ownsRuntime || _ownedPlatformSource is not null) return; + try + { + _ownedRepositoryClient ??= GalaxyRepositoryClient.Create( + await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false)); + _ownedPlatformSource = + new GatewayGalaxyPlatformSource(_ownedRepositoryClient, _options.Repository.DiscoverPageSize); + } + catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + _logger.LogWarning(ex, + "GalaxyDriver {InstanceId} could not build the platform query — per-platform host status stays unseeded until the next rediscovery.", _driverInstanceId); } - - await StartDeployWatcherAsync(cancellationToken).ConfigureAwait(false); - _logger.LogInformation( - "GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}", - _driverInstanceId, _options.Gateway.Endpoint, _options.MxAccess.ClientName); - _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null); } + /// + /// Ask for a background probe seed (init and every deploy-watcher rediscovery). Requests + /// coalesce onto one worker; each worker pass serves the runtime that is live when the pass + /// STARTS, and a pass whose fetch completes after a teardown is discarded — so a request made + /// under a new runtime always gets a pass of its own. + /// + private void RequestProbeSeed(string reason) + { + if (_runtimeStopping || _probeWatcher is null) return; + _seedReason = reason; + if (Interlocked.Increment(ref _reseedRequests) != 1) return; // the running worker will serve it + + _ = Task.Run(async () => + { + int served; + do + { + served = Volatile.Read(ref _reseedRequests); + var generation = Volatile.Read(ref _runtimeGeneration); + var seedCts = Volatile.Read(ref _seedCts); + if (!_runtimeStopping && seedCts is not null) + { + await SeedProbePlatformsAsync(_seedReason, generation, seedCts.Token).ConfigureAwait(false); + } + } + while (Interlocked.Add(ref _reseedRequests, -served) != 0); + }); + } + + /// + /// One seed pass: fetch the platform / engine set and hand it to the probe watcher of the + /// runtime identified by . Never throws — a failed or cancelled + /// fetch logs and leaves the watcher as it was (the next rediscovery retries), and a result + /// that lands after that runtime was torn down is discarded. + /// + private async Task SeedProbePlatformsAsync(string reason, int generation, CancellationToken cancellationToken) + { + var watcher = _probeWatcher; + var source = _platformSource ?? _ownedPlatformSource; + if (watcher is null || source is null) return; + + try + { + var platforms = await source.GetPlatformTagNamesAsync(cancellationToken).ConfigureAwait(false); + if (Volatile.Read(ref _runtimeGeneration) != generation || _runtimeStopping + || !ReferenceEquals(watcher, _probeWatcher)) + { + _logger.LogDebug( + "GalaxyDriver {InstanceId} discarded a platform seed ({Reason}) — its runtime was torn down.", + _driverInstanceId, reason); + return; + } + await watcher.SyncPlatformsAsync(platforms, cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "GalaxyDriver {InstanceId} probe watcher seeded with {Count} platform(s) ({Reason}).", + _driverInstanceId, platforms.Count, reason); + } + catch (Exception ex) when (cancellationToken.IsCancellationRequested + || Volatile.Read(ref _runtimeGeneration) != generation) + { + _logger.LogDebug(ex, "GalaxyDriver {InstanceId} platform seed ({Reason}) abandoned by a teardown.", + _driverInstanceId, reason); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "GalaxyDriver {InstanceId} platform seed failed ({Reason}) — per-platform host status stays as it was until the next rediscovery.", + _driverInstanceId, reason); + } + } + + /// Deploy-watcher rediscovery: a redeploy may add or remove platforms / engines. + private void ReseedProbePlatformsOnRediscovery() => RequestProbeSeed("rediscovery"); + + /// + /// Register only while is still the + /// live runtime. Atomic with the teardown's registry Clear (both under ). + /// + private bool TryRegisterForGeneration(long subscriptionId, IReadOnlyList bindings, int generation) + { + lock (_registryGate) + { + if (Volatile.Read(ref _runtimeGeneration) != generation) return false; + _subscriptions.Register(subscriptionId, bindings); + return true; + } + } + + private void ClearRegistry() + { + lock (_registryGate) _subscriptions.Clear(); + } + + /// + /// Best-effort release of the live handles a SubscribeBulk produced for a call that then lost + /// its runtime — the subscriber may already be dead, so a failure is logged and swallowed. + /// + private async Task ReleaseOrphanedHandlesAsync(IGalaxySubscriber subscriber, IReadOnlyList bindings) + { + var live = bindings.Where(b => b.ItemHandle > 0).Select(b => b.ItemHandle).ToArray(); + if (live.Length == 0) return; + try { await subscriber.UnsubscribeBulkAsync(live, CancellationToken.None).ConfigureAwait(false); } + catch (Exception ex) + { + _logger.LogWarning(ex, + "GalaxyDriver {InstanceId} could not release {Count} handle(s) orphaned by a teardown.", + _driverInstanceId, live.Length); + } + } + + private static InvalidOperationException TornDownDuringSubscribe() => new( + "GalaxyDriver runtime was torn down while the subscribe was in flight — retry after the next InitializeAsync."); + + // The watcher publishes its probe handles into the driver's registry (reserved id) so the + // EventPump's fan-out reaches OnPumpDataChange for ScanState events — without that every + // probe event was dropped before it got anywhere near the watcher. + private PerPlatformProbeWatcher BuildProbeWatcher(IGalaxySubscriber subscriber) => + new(subscriber, _hostStatuses, _subscriptions, _logger, + bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs); + /// /// Build the production gw client + session + per-capability runtime components /// from _options. Sets up the reconnect supervisor's reopen / replay @@ -389,9 +658,45 @@ public sealed class GalaxyDriver /// private async Task BuildProductionRuntimeAsync(CancellationToken cancellationToken) { + try + { + await BuildProductionRuntimeCoreAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + // A failed connect must not leave a half-built runtime behind: the watchdog retries + // InitializeAsync every 10 s, and the seam check above would see the orphaned client + + // session as "already built" — or, worse, rebuild on top of them. Release what was + // created, and land the failed state before AND after the teardown (IDriver remark). + _health = new DriverHealth(DriverState.Faulted, _health.LastSuccessfulRead, ex.Message); + await TeardownRuntimeAsync(final: false, CancellationToken.None).ConfigureAwait(false); + _health = new DriverHealth(DriverState.Faulted, _health.LastSuccessfulRead, ex.Message); + throw; + } + } + + private async Task BuildProductionRuntimeCoreAsync(CancellationToken cancellationToken) + { + // A pump that slipped in between a teardown and this rebuild (a late SubscribeAsync) is bound + // to the dead subscriber; it must never be reused as this runtime's pump. + EventPump? stale; + lock (_pumpLock) { stale = _eventPump; _eventPump = null; } + if (stale is not null) + { + stale.OnDataChange -= OnPumpDataChange; + try { await stale.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "Stale EventPump dispose failed before runtime rebuild"); } + } + // Unconditionally: whatever the registry holds names handles of a session this rebuild replaces. + ClearRegistry(); + + _ownsRuntime = true; var clientOptions = await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false); _ownedMxClient = MxGatewayClient.Create(clientOptions); - _ownedMxSession = new GalaxyMxSession(_options.MxAccess, _logger); + _ownedMxSession = new GalaxyMxSession(_options.MxAccess, _logger) + { + OpenAndRegisterOverrideForTests = SessionOpenOverrideForTests, + }; await _ownedMxSession.ConnectAsync(clientOptions, cancellationToken).ConfigureAwait(false); // PR 6.1 — wrap the gw-facing seams in tracing decorators so every Subscribe / @@ -424,9 +729,7 @@ public sealed class GalaxyDriver _supervisor.StateChanged += OnSupervisorStateChanged; - _probeWatcher = new PerPlatformProbeWatcher( - _subscriber, _hostStatuses, _logger, - bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs); + _probeWatcher = BuildProbeWatcher(_subscriber); // Wire the alarm acknowledger + feed to the live gateway client. Both are // session-less — the gateway serves alarms from an always-on central monitor — @@ -436,6 +739,143 @@ public sealed class GalaxyDriver _ownedMxClient.StreamAlarmsAsync, _logger, _options.MxAccess.ClientName); } + /// + /// Stop every background loop and release every gateway resource this driver holds — the + /// teardown half shared by ( = false: + /// a later rebuilds) and + /// ( = true). Before this existed, ShutdownAsync was a + /// no-op and everything below outlived the driver on every respawn, removal and + /// /raw browse: the gateway accumulated live MX sessions and advised items, and + /// this process accumulated channels and stream loops. + /// + /// + /// Ordering matters: + /// + /// The supervisor goes first and is awaited, so no recovery loop can reopen the + /// session or start a pump while the rest is being released. + /// The probe watcher unsubscribes while the session is still open (its handles + /// die with the session anyway, but a clean release keeps the gateway's advise + /// count honest). + /// Streams (pump, alarm feed, deploy watcher) are cancelled and awaited. + /// The session closes (gateway CloseSession), then both owned channels. + /// + /// Every step is best-effort: one failing release never prevents the next. + /// + private async Task TeardownRuntimeAsync(bool final, CancellationToken cancellationToken) + { + _runtimeStopping = true; + Interlocked.Increment(ref _runtimeGeneration); + Interlocked.Exchange(ref _seedCts, null)?.Cancel(); // a seed in flight belongs to this runtime + try + { + var supervisor = _supervisor; + _supervisor = null; + if (supervisor is not null) + { + supervisor.StateChanged -= OnSupervisorStateChanged; + try { await supervisor.StopAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "ReconnectSupervisor stop failed"); } + } + + var deployWatcher = _deployWatcher; + _deployWatcher = null; + if (deployWatcher is not null) + { + deployWatcher.OnRediscoveryNeeded -= OnDeployWatcherRediscoveryNeeded; + try { await deployWatcher.StopAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "DeployWatcher stop failed"); } + } + + var probeWatcher = _probeWatcher; + _probeWatcher = null; + if (probeWatcher is not null) + { + try { await probeWatcher.StopAsync(cancellationToken).ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "ProbeWatcher stop failed"); } + } + + try { _transportForwarder?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "Transport forwarder dispose failed"); } + _transportForwarder = null; + + EventPump? pump; + lock (_pumpLock) { pump = _eventPump; _eventPump = null; } + if (pump is not null) + { + pump.OnDataChange -= OnPumpDataChange; + try { await pump.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "EventPump dispose failed"); } + } + // Every item handle the registry knows died with the pump's session; the host + // re-subscribes (fresh ids) once the next InitializeAsync reports Connected. + ClearRegistry(); + + // The alarm feed: owned → disposed and nulled so InitializeAsync builds a fresh one; + // injected → detached (the injector owns its lifetime) unless this is the final dispose. + IGalaxyAlarmFeed? alarmFeed; + bool releaseFeed; + lock (_alarmFeedLock) + { + alarmFeed = _alarmFeed; + releaseFeed = alarmFeed is not null && (_ownsRuntime || final); + if (alarmFeed is not null && _alarmFeedWired) + { + alarmFeed.OnAlarmTransition -= OnAlarmFeedTransition; + _alarmFeedWired = false; + } + if (releaseFeed) _alarmFeed = null; + } + if (releaseFeed) + { + try { await alarmFeed!.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "Alarm feed dispose failed"); } + } + + if (_ownedMxSession is not null) + { + var session = _ownedMxSession; + _ownedMxSession = null; + // Bounded close: the caller's token plus SessionCloseDeadline, so a blackholed gateway + // cannot park this (and the host's PostStop) for the SDK's 30 s call timeout. + using var closeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + closeCts.CancelAfter(SessionCloseDeadline); + try { await session.CloseAsync(closeCts.Token).ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "MxSession close failed"); } + try { await session.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "MxSession dispose failed"); } + } + + if (_ownedMxClient is not null) + { + try { await _ownedMxClient.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "MxClient dispose failed"); } + _ownedMxClient = null; + } + + if (_ownedRepositoryClient is not null) + { + try { await _ownedRepositoryClient.DisposeAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "RepositoryClient dispose failed"); } + _ownedRepositoryClient = null; + } + + if (_ownsRuntime || final) + { + // Null the seams so the next InitializeAsync rebuilds (production) — injected seams + // are only dropped on the final dispose. + _subscriber = null; + _dataWriter = null; + _hierarchySource = null; + _alarmAcknowledger = null; + _ownedPlatformSource = null; + _ownsRuntime = false; + } + } + finally + { + _runtimeStopping = false; + } + } + /// /// Reopen callback for : recreate the gw session. After a /// gateway restart the existing session handle is faulted / stale but non-null, so a plain @@ -465,6 +905,9 @@ public sealed class GalaxyDriver /// private async Task ReplayAsync(CancellationToken cancellationToken) { + // A teardown cancels the supervisor's cycle between reopen and replay; a replay that ran on + // anyway would start a fresh pump underneath the shutdown. + cancellationToken.ThrowIfCancellationRequested(); if (_subscriber is null) return; var entries = _subscriptions.SnapshotEntries(); if (entries.Count == 0) return; @@ -488,7 +931,8 @@ public sealed class GalaxyDriver // completed loop. Recreate the pump so the replayed subscriptions have a consumer. RestartEventPumpForReplay(); - var tagCount = await ResubscribeAllAsync(entries, cancellationToken).ConfigureAwait(false); + var tagCount = await ResubscribeAllAsync(entries, Volatile.Read(ref _runtimeGeneration), cancellationToken) + .ConfigureAwait(false); _logger.LogInformation( "GalaxyDriver {InstanceId} replay completed — {SubCount} subscriptions, {TagCount} tags re-subscribed", @@ -502,14 +946,37 @@ public sealed class GalaxyDriver /// replay must recreate (its loop died with the stream) and a gap must NOT touch (the /// stream carrying the sentinel is still healthy and still delivering). /// + /// The subscriptions to re-issue. + /// + /// The runtime generation this re-subscribe belongs to. Re-checked before every + /// : a teardown bumps it and clears the registry, + /// and a Rebind landing after that would resurrect dead handles into the next runtime. + /// + /// Cancellation token. /// The number of tag references re-subscribed. private async Task ResubscribeAllAsync( IReadOnlyList<(long SubscriptionId, IReadOnlyList Bindings)> entries, + int generation, CancellationToken cancellationToken) { var tagCount = 0; foreach (var (subscriptionId, oldBindings) in entries) { + if (Volatile.Read(ref _runtimeGeneration) != generation) + { + _logger.LogDebug("GalaxyDriver {InstanceId} re-subscribe abandoned — runtime was torn down.", _driverInstanceId); + return tagCount; + } + + if (subscriptionId == SubscriptionRegistry.ProbeSubscriptionId && _probeWatcher is { } probeWatcher) + { + // The probe watcher owns its handles: it re-subscribes its platforms, records the + // fresh handles and rebinds the reserved registry entry itself, so ScanState host + // status keeps updating after a reconnect (it used to go permanently deaf). + tagCount += await probeWatcher.ResubscribeAsync(cancellationToken).ConfigureAwait(false); + continue; + } + var refs = oldBindings .Select(b => b.FullReference) .Distinct(StringComparer.OrdinalIgnoreCase) @@ -531,8 +998,13 @@ public sealed class GalaxyDriver } // Rebind so the EventPump fan-out reverse map points at the post-reconnect - // handles; otherwise events on the new handles miss every subscription. - _subscriptions.Rebind(subscriptionId, newBindings); + // handles; otherwise events on the new handles miss every subscription. Not after a + // teardown: the fresh handles belong to a subscriber that is already being released. + lock (_registryGate) + { + if (Volatile.Read(ref _runtimeGeneration) != generation) return tagCount; + _subscriptions.Rebind(subscriptionId, newBindings); + } tagCount += refs.Length; } @@ -581,9 +1053,9 @@ public sealed class GalaxyDriver } // EnsureEventPumpStarted creates + starts a fresh pump under the lock. Skip the - // recreate if the driver is being disposed — Dispose already tore the pump down - // and a fresh one here would leak past the driver's lifetime. - if (_disposed) return; + // recreate if the driver is being disposed or its runtime torn down — the teardown + // already released the pump and a fresh one here would leak past it. + if (_disposed || _runtimeStopping) return; EnsureEventPumpStarted(); } @@ -631,28 +1103,57 @@ public sealed class GalaxyDriver private async Task StartDeployWatcherAsync(CancellationToken cancellationToken) { if (!_options.Repository.WatchDeployEvents) return; - if (_ownedRepositoryClient is null && _hierarchySource is null) return; - // Reuse the lazily-built repository client (DiscoverAsync constructs it on demand). - // If discovery hasn't run yet, build the client here so the watcher has a target. - // Build under a null-check (not ??=) so if BuildDefaultHierarchySourceAsync later - // runs it reuses this client rather than overwriting the field and leaking the - // first instance — the client-options build is now async (secret: arm). - _ownedRepositoryClient ??= ZB.MOM.WW.MxGateway.Client.GalaxyRepositoryClient.Create( - await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false)); + var source = _deployWatchSource; + if (source is null) + { + // Seam mode without an injected watch source has no gateway to watch. (Production + // always owns its runtime here.) The old guard returned whenever BOTH the repository + // client and the hierarchy source were null — which on a production runtime instance + // is every first InitializeAsync, since discovery runs on the /raw picker's throwaway + // instance, not here — so the watcher never started and the re-browse chip never + // fired for a live driver. + if (!_ownsRuntime) return; - var source = new GatewayGalaxyDeployWatchSource(_ownedRepositoryClient); - _deployWatcher = new DeployWatcher(source, _logger); - _deployWatcher.OnRediscoveryNeeded += (_, args) => OnRediscoveryNeeded?.Invoke(this, args); + // Reuse the lazily-built repository client (DiscoverAsync constructs it on demand). + // Build under a null-check (not an overwrite) so if BuildDefaultHierarchySourceAsync + // later runs it reuses this client rather than leaking the first instance. + _ownedRepositoryClient ??= GalaxyRepositoryClient.Create( + await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false)); + source = new GatewayGalaxyDeployWatchSource(_ownedRepositoryClient); + } + + // One watcher per driver: a previous one (an InitializeAsync without a ShutdownAsync in + // between) is stopped before its replacement starts — every live InitializeAsync used to + // stack another watch-stream loop, each raising OnRediscoveryNeeded per real deploy. + var previous = _deployWatcher; + _deployWatcher = null; + if (previous is not null) + { + previous.OnRediscoveryNeeded -= OnDeployWatcherRediscoveryNeeded; + try { await previous.StopAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "Previous DeployWatcher stop failed — replacing it anyway."); } + } + + var watcher = new DeployWatcher(source, _logger); + watcher.OnRediscoveryNeeded += OnDeployWatcherRediscoveryNeeded; + _deployWatcher = watcher; // StartAsync schedules the background loop and returns Task.CompletedTask immediately. // It throws InvalidOperationException synchronously if called twice (programming error). // Don't discard the return value — observe any synchronous throw. - var startTask = _deployWatcher.StartAsync(CancellationToken.None); + var startTask = watcher.StartAsync(CancellationToken.None); // The task is already completed (StartAsync is synchronous); surface any synchronous fault. if (startTask.IsFaulted) startTask.GetAwaiter().GetResult(); } + private void OnDeployWatcherRediscoveryNeeded(object? sender, RediscoveryEventArgs args) + { + OnRediscoveryNeeded?.Invoke(this, args); + // A redeploy may add or remove platforms / engines; refresh the ScanState probe set. + ReseedProbePlatformsOnRediscovery(); + } + /// public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) { @@ -731,13 +1232,32 @@ public sealed class GalaxyDriver && a.Reconnect == b.Reconnect && a.ProbeTimeoutSeconds == b.ProbeTimeoutSeconds; + /// + /// Stop the driver and release the gateway runtime: supervisor, deploy watcher, probe + /// watcher, event pump, alarm feed, MX session (gateway CloseSession) and both owned + /// gRPC channels. The seams are nulled afterwards, so a later + /// on this same object — the host's comms-loss recovery and the way ReinitializeAsync + /// is sequenced — rebuilds a fresh runtime instead of reusing a dead one. The driver object + /// itself stays usable; is the final, idempotent teardown. + /// /// - public Task ShutdownAsync(CancellationToken cancellationToken) + public async Task ShutdownAsync(CancellationToken cancellationToken) { - if (_disposed) return Task.CompletedTask; + if (_disposed) return; _logger.LogInformation("GalaxyDriver {InstanceId} shutting down", _driverInstanceId); - _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null); - return Task.CompletedTask; + await _lifecycleGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_disposed) return; + await TeardownRuntimeAsync(final: false, cancellationToken).ConfigureAwait(false); + // Terminal health lands AFTER the teardown (IDriver remark): a loop unwinding during the + // release could otherwise overwrite it with the state it was reporting when cancelled. + _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null); + } + finally + { + _lifecycleGate.Release(); + } } /// @@ -786,11 +1306,12 @@ public sealed class GalaxyDriver // keyed by RawPath, matching what WriteAsync resolves against. var discoverer = new GalaxyDiscoverer( source, attributeRef => _rawTagByAttributeRef.GetValueOrDefault(attributeRef)); - await discoverer.DiscoverAsync(capturingBuilder, cancellationToken).ConfigureAwait(false); + // One sweep: the discoverer hands back the hierarchy it materialised, and the platform + // selection below reads that instead of asking the gateway for the whole galaxy again. + var hierarchy = await discoverer.DiscoverAsync(capturingBuilder, cancellationToken).ConfigureAwait(false); if (_probeWatcher is not null) { - var hierarchy = await source.GetHierarchyAsync(cancellationToken).ConfigureAwait(false); var platforms = hierarchy .Where(o => o.TemplateChain.Any(t => string.Equals(t, "$WinPlatform", StringComparison.OrdinalIgnoreCase) @@ -862,6 +1383,8 @@ public sealed class GalaxyDriver deadlineCts.CancelAfter(ReadViaSubscribeDeadline); var cancellationToken = deadlineCts.Token; + var generation = Volatile.Read(ref _runtimeGeneration); // see SubscribeAsync + var readSubscriber = _subscriber!; var pump = EnsureEventPumpStarted(); var subscriptionId = _subscriptions.NextSubscriptionId(); @@ -895,7 +1418,7 @@ public sealed class GalaxyDriver // The RPC gets only the CALLER's token: the internal deadline bounds our wait, not the RPC, so a slow // gateway can never surface an OperationCanceledException to a caller that did not cancel, and the // handles it eventually creates are known (and unsubscribed below) rather than lost to a cancelled call. - var subscribeTask = _subscriber!.SubscribeBulkAsync(fullReferences, bufferedIntervalMs, callerToken); + var subscribeTask = readSubscriber.SubscribeBulkAsync(fullReferences, bufferedIntervalMs, callerToken); try { results = await subscribeTask.WaitAsync(cancellationToken).ConfigureAwait(false); @@ -904,7 +1427,7 @@ public sealed class GalaxyDriver { // Internal deadline fired while the subscribe RPC was still in flight. pump.OnDataChange -= handler; - var subscriber = _subscriber!; + var subscriber = readSubscriber; var logger = _logger; _ = subscribeTask.ContinueWith(t => { @@ -956,7 +1479,12 @@ public sealed class GalaxyDriver ServerTimestampUtc: DateTime.UtcNow)); } } - _subscriptions.Register(subscriptionId, bindings); + if (!TryRegisterForGeneration(subscriptionId, bindings, generation)) + { + pump.OnDataChange -= handler; + await ReleaseOrphanedHandlesAsync(readSubscriber, bindings).ConfigureAwait(false); + throw TornDownDuringSubscribe(); + } try { @@ -995,7 +1523,7 @@ public sealed class GalaxyDriver { try { - await _subscriber!.UnsubscribeBulkAsync(liveHandles, CancellationToken.None) + await readSubscriber.UnsubscribeBulkAsync(liveHandles, CancellationToken.None) .ConfigureAwait(false); } catch (Exception ex) @@ -1056,6 +1584,10 @@ public sealed class GalaxyDriver "Call InitializeAsync and wait for a successful connection before subscribing."); } + // Captured before the pump/stopping check: a teardown that starts after this point bumps the + // generation, so the Register below can tell its SubscribeBulk straddled a teardown. + var generation = Volatile.Read(ref _runtimeGeneration); + var subscriber = _subscriber; var pump = EnsureEventPumpStarted(); var subscriptionId = _subscriptions.NextSubscriptionId(); @@ -1063,7 +1595,7 @@ public sealed class GalaxyDriver { // Empty subscriptions register but never bind anything — keeps Unsubscribe // symmetric for callers that conditionally add tags later. - _subscriptions.Register(subscriptionId, []); + if (!TryRegisterForGeneration(subscriptionId, [], generation)) throw TornDownDuringSubscribe(); return new GalaxySubscriptionHandle(subscriptionId); } @@ -1079,7 +1611,7 @@ public sealed class GalaxyDriver // and infrastructure callers (probe watcher, deploy watcher) hit the fallback. var requested = (int)Math.Max(0, publishingInterval.TotalMilliseconds); var bufferedIntervalMs = requested > 0 ? requested : _options.MxAccess.PublishingIntervalMs; - var results = await _subscriber + var results = await subscriber .SubscribeBulkAsync(dialledRefs, bufferedIntervalMs, cancellationToken) .ConfigureAwait(false); @@ -1107,7 +1639,13 @@ public sealed class GalaxyDriver } } - _subscriptions.Register(subscriptionId, bindings); + if (!TryRegisterForGeneration(subscriptionId, bindings, generation)) + { + // The teardown cleared the registry while SubscribeBulk was in flight; registering now + // would carry dead handles into the next runtime. Release what we got and fail the call. + await ReleaseOrphanedHandlesAsync(subscriber, bindings).ConfigureAwait(false); + throw TornDownDuringSubscribe(); + } _ = pump; // keep the pump alive for the subscription's lifetime return new GalaxySubscriptionHandle(subscriptionId); } @@ -1152,6 +1690,15 @@ public sealed class GalaxyDriver { lock (_pumpLock) { + // Checked BEFORE the reuse: the flag is set before the teardown takes the pump under this + // same lock, so while it is set the live pump belongs to a dying subscriber. Handing it out + // would register handles into a registry the teardown is about to clear; creating a new one + // would leave a pump on the dead subscriber for the next InitializeAsync to inherit. + if (_runtimeStopping) + { + throw new InvalidOperationException( + "GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync."); + } if (_eventPump is not null) return _eventPump; _eventPump = new EventPump( _subscriber!, _subscriptions, _logger, @@ -1192,7 +1739,7 @@ public sealed class GalaxyDriver /// private void OnEventPumpReplayGap(ReplayGap gap) { - if (_subscriber is null) return; + if (_subscriber is null || _runtimeStopping) return; if (!_options.Reconnect.ReplayOnSessionLost) { @@ -1212,6 +1759,7 @@ public sealed class GalaxyDriver return; } + var generation = Volatile.Read(ref _runtimeGeneration); _ = Task.Run(async () => { try @@ -1219,7 +1767,7 @@ public sealed class GalaxyDriver var entries = _subscriptions.SnapshotEntries(); if (entries.Count == 0) return; - var tagCount = await ResubscribeAllAsync(entries, CancellationToken.None).ConfigureAwait(false); + var tagCount = await ResubscribeAllAsync(entries, generation, CancellationToken.None).ConfigureAwait(false); _logger.LogInformation( "GalaxyDriver {InstanceId} replay-gap re-snapshot completed — {SubCount} subscriptions, {TagCount} tags re-subscribed", _driverInstanceId, entries.Count, tagCount); @@ -1249,6 +1797,14 @@ public sealed class GalaxyDriver /// private void OnEventPumpStreamFault(Exception cause) { + if (_runtimeStopping) + { + // The pump is being torn down with the rest of the runtime — a fault it reports on the + // way out is shutdown noise, not something to recover from. + _logger.LogDebug(cause, "GalaxyDriver {InstanceId} event stream ended during runtime teardown.", _driverInstanceId); + return; + } + var supervisor = _supervisor; if (supervisor is null) { @@ -1464,6 +2020,24 @@ public sealed class GalaxyDriver /// private void OnPumpDataChange(object? sender, DataChangeEventArgs args) { + // Anything the gateway pushes on the stream proves it is alive and serving this session — + // the comms-loss watchdog's evidence (G-11: the stamp used to move only at init and on a + // supervisor Healthy transition, so a 90 s Degraded tripped a re-init under flowing data). + StampLiveness(); + + // Probe traffic rides the reserved registry entry and stays in the attributeRef domain — + // ScanState attributes are discovered platform attributes, never authored RawPaths. It is + // the watcher's alone: the host never subscribed it and must not see it on OnDataChange. + if (args.SubscriptionHandle is GalaxySubscriptionHandle { SubscriptionId: SubscriptionRegistry.ProbeSubscriptionId }) + { + // The probe decoder takes a raw quality byte. Recover it via the canonical + // StatusCodeMap.ToQualityCategoryByte helper so the mapping lives in one + // place next to its inverse (FromQualityByte) and cannot desync silently. + var qualityByte = StatusCodeMap.ToQualityCategoryByte(args.Snapshot.StatusCode); + _probeWatcher?.OnProbeValueChanged(args.FullReference, args.Snapshot.Value, qualityByte); + return; + } + // v3: the pump fans out keyed by the dialled attributeRef; the server correlates values to nodes // by RawPath (the node's FullName). Reverse-map before surfacing to public OnDataChange consumers. // Identity on a miss, so a live-browse tag surfaces under its attributeRef unchanged. @@ -1474,18 +2048,25 @@ public sealed class GalaxyDriver publicArgs = args with { FullReference = rawPath }; } OnDataChange?.Invoke(this, publicArgs); + } - // Probe watching stays in the attributeRef domain — ScanState attributes are discovered platform - // attributes, never authored RawPaths, so use the untranslated reference here. - if (_probeWatcher is not null - && args.FullReference.EndsWith(PerPlatformProbeWatcher.ProbeSuffix, StringComparison.OrdinalIgnoreCase)) - { - // The probe decoder takes a raw quality byte. Recover it via the canonical - // StatusCodeMap.ToQualityCategoryByte helper so the mapping lives in one - // place next to its inverse (FromQualityByte) and cannot desync silently. - var qualityByte = StatusCodeMap.ToQualityCategoryByte(args.Snapshot.StatusCode); - _probeWatcher.OnProbeValueChanged(args.FullReference, args.Snapshot.Value, qualityByte); - } + /// Minimum gap between two LastSuccessfulRead stamps from stream data. Internal so tests can widen it. + internal TimeSpan LivenessStampThrottle { get; set; } = TimeSpan.FromSeconds(1); + + private long _lastLivenessStampTicks; + + /// + /// Advance LastSuccessfulRead from inbound stream data, at most once per + /// — this runs on the pump's dispatch loop for every + /// event, so a 50k-tag galaxy must not pay a health-record allocation per value. + /// + private void StampLiveness() + { + var now = DateTime.UtcNow; + var last = Volatile.Read(ref _lastLivenessStampTicks); + if (now.Ticks - last < LivenessStampThrottle.Ticks) return; + Volatile.Write(ref _lastLivenessStampTicks, now.Ticks); + _health = _health with { LastSuccessfulRead = now }; } /// @@ -1503,7 +2084,8 @@ public sealed class GalaxyDriver _ownedRepositoryClient ??= GalaxyRepositoryClient.Create( await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false)); return new TracedGalaxyHierarchySource( - new GatewayGalaxyHierarchySource(_ownedRepositoryClient), _options.MxAccess.ClientName); + new GatewayGalaxyHierarchySource(_ownedRepositoryClient, _options.Repository.DiscoverPageSize), + _options.MxAccess.ClientName); } /// @@ -1517,54 +2099,19 @@ public sealed class GalaxyDriver public async ValueTask DisposeAsync() { if (_disposed) return; - _disposed = true; - - // Synchronous sub-components first — none of these block. - try { _deployWatcher?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "DeployWatcher dispose failed"); } - try { _supervisor?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "ReconnectSupervisor dispose failed"); } - try { _probeWatcher?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "ProbeWatcher dispose failed"); } - try { _transportForwarder?.Dispose(); } catch (Exception ex) { _logger.LogWarning(ex, "Transport forwarder dispose failed"); } - - // Async sub-components: await each so we don't block a thread-pool thread - // on a slow shutdown (e.g. EventPump draining its channel, gRPC stream closing). - EventPump? pump; - lock (_pumpLock) { pump = _eventPump; _eventPump = null; } - if (pump is not null) + await _lifecycleGate.WaitAsync().ConfigureAwait(false); + try { - try { await pump.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "EventPump dispose failed"); } + if (_disposed) return; + _disposed = true; + // Same release sequence as ShutdownAsync, plus the injected seams this instance will + // never need again. Idempotent: a ShutdownAsync before this finds nothing left to do. + await TeardownRuntimeAsync(final: true, CancellationToken.None).ConfigureAwait(false); } - - IGalaxyAlarmFeed? alarmFeed; - lock (_alarmFeedLock) { alarmFeed = _alarmFeed; _alarmFeed = null; } - if (alarmFeed is not null) + finally { - try { await alarmFeed.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "Alarm feed dispose failed"); } + _lifecycleGate.Release(); } - - if (_ownedMxSession is not null) - { - try { await _ownedMxSession.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "MxSession dispose failed"); } - _ownedMxSession = null; - } - - if (_ownedMxClient is not null) - { - try { await _ownedMxClient.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "MxClient dispose failed"); } - _ownedMxClient = null; - } - - if (_ownedRepositoryClient is not null) - { - try { await _ownedRepositoryClient.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "RepositoryClient dispose failed"); } - _ownedRepositoryClient = null; - } - - _hierarchySource = null; } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs index a4010d07..cfa757b8 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverProbe.cs @@ -54,12 +54,18 @@ public sealed class GalaxyDriverProbe : IDriverProbe if (string.IsNullOrWhiteSpace(host) || port <= 0) return new(false, "Config has no host/port to probe.", null); + // The config's own ProbeTimeoutSeconds caps the probe deadline. It can only tighten the + // caller's budget, never extend it — the caller's token fires at `timeout` regardless. + timeout = EffectiveTimeout(timeout, opts.ProbeTimeoutSeconds); + using var probeCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + probeCts.CancelAfter(timeout); + // --- TCP preflight: fast-fail for closed ports / unreachable hosts --- var sw = Stopwatch.StartNew(); try { using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - await socket.ConnectAsync(host, port, ct); + await socket.ConnectAsync(host, port, probeCts.Token); } catch (SocketException ex) { @@ -86,10 +92,7 @@ public sealed class GalaxyDriverProbe : IDriverProbe { client = GalaxyRepositoryClient.Create(BuildProbeClientOptions(opts.Gateway, timeout)); - using var deadlineCts = CancellationTokenSource.CreateLinkedTokenSource(ct); - deadlineCts.CancelAfter(timeout); - - await client.TestConnectionAsync(deadlineCts.Token).ConfigureAwait(false); + await client.TestConnectionAsync(probeCts.Token).ConfigureAwait(false); sw.Stop(); // No RpcException ⇒ the gateway answered the gRPC call successfully. @@ -157,6 +160,26 @@ public sealed class GalaxyDriverProbe : IDriverProbe _ => (false, $"Reachable at {host}:{port} but gateway gRPC handshake failed: {code}"), }; + /// + /// The deadline the probe actually runs under: the caller's budget, tightened by the + /// config's when that is positive and + /// shorter. The knob was authored by the AdminUI and parsed by the factory but consumed by + /// nothing — the probe always ran on the caller's budget alone. A non-positive caller budget + /// is floored to 1 s before the comparison (the same floor + /// applies to the gRPC call budget), so the probe runs briefly rather than cancelling before + /// it starts; a longer config value never raises it above that floor. + /// + /// The budget the host handed the probe. + /// The config's ProbeTimeoutSeconds; ignored when not positive. + /// The effective probe deadline. + internal static TimeSpan EffectiveTimeout(TimeSpan callerTimeout, int configuredSeconds) + { + var caller = callerTimeout > TimeSpan.Zero ? callerTimeout : TimeSpan.FromSeconds(1); + if (configuredSeconds <= 0) return caller; + var configured = TimeSpan.FromSeconds(configuredSeconds); + return configured < caller ? configured : caller; + } + /// /// Builds the gRPC client options for the probe ping from the gateway config WITHOUT /// resolving the API-key secret ref (the host owns secret resolution; the probe sends the diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs index b766e38b..64080d52 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs @@ -38,12 +38,19 @@ public sealed class PerPlatformProbeWatcher : IDisposable private readonly ILogger _logger; private readonly int _bufferedUpdateIntervalMs; + // The driver's fan-out registry. The EventPump resolves every inbound event through it, so + // the probe handles must be registered there (under SubscriptionRegistry.ProbeSubscriptionId) + // or every ScanState event is dropped as "stale after unsubscribe". Null when the watcher is + // driven directly (unit tests of the decoding table). + private readonly SubscriptionRegistry? _registry; + // Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe; // we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call. private readonly ConcurrentDictionary _itemHandlesByPlatform = new(StringComparer.OrdinalIgnoreCase); private readonly Lock _syncLock = new(); - private bool _disposed; + // Written under _syncLock by StopAsync; read under it wherever a write-back must not outlive a stop. + private volatile bool _disposed; /// Initializes a new instance of the PerPlatformProbeWatcher class. /// The Galaxy subscriber for managing probe subscriptions. @@ -55,9 +62,30 @@ public sealed class PerPlatformProbeWatcher : IDisposable HostStatusAggregator aggregator, ILogger? logger = null, int bufferedUpdateIntervalMs = 0) + : this(subscriber, aggregator, registry: null, logger, bufferedUpdateIntervalMs) + { + } + + /// + /// Driver-side ctor: the watcher publishes its probe handles into + /// under so the + /// fans ScanState events out to . + /// + /// The Galaxy subscriber for managing probe subscriptions. + /// The host status aggregator for tracking platform connectivity. + /// The driver's subscription registry; null to keep the handles private. + /// Optional logger for diagnostic messages. + /// Buffered update interval in milliseconds; must be >= 0. + internal PerPlatformProbeWatcher( + IGalaxySubscriber subscriber, + HostStatusAggregator aggregator, + SubscriptionRegistry? registry, + ILogger? logger = null, + int bufferedUpdateIntervalMs = 0) { _subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber)); _aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator)); + _registry = registry; _logger = logger ?? NullLogger.Instance; if (bufferedUpdateIntervalMs < 0) { @@ -67,6 +95,103 @@ public sealed class PerPlatformProbeWatcher : IDisposable _bufferedUpdateIntervalMs = bufferedUpdateIntervalMs; } + /// Current platform → probe-handle bindings, as the registry sees them. Test seam. + internal IReadOnlyList CurrentBindings => BuildBindings(); + + private IReadOnlyList BuildBindings() => + [.. _itemHandlesByPlatform.Select(kvp => new TagBinding(kvp.Key + ProbeSuffix, kvp.Value))]; + + /// + /// Mirror the tracked handle set into the driver registry (no-op without one, or once stopped — + /// a stopped watcher must never write its entry back into a registry the teardown cleared). + /// + private void PublishBindings() + { + lock (_syncLock) PublishBindingsLocked(); + } + + private void PublishBindingsLocked() + { + if (_registry is null || _disposed) return; + var bindings = BuildBindings(); + if (bindings.Count == 0) _registry.Remove(SubscriptionRegistry.ProbeSubscriptionId); + else _registry.Upsert(SubscriptionRegistry.ProbeSubscriptionId, bindings); + } + + /// + /// Re-issue SubscribeBulk for every tracked platform and replace the handles — the + /// reconnect replay path, where the gateway session was recreated and every pre-reconnect + /// handle is dead. Platforms the gateway now rejects keep a 0 handle (no events expected). + /// + /// A cancellation token to cancel the operation. + /// The number of probe references re-subscribed. + public async Task ResubscribeAsync(CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(_disposed, this); + var platforms = _itemHandlesByPlatform.Keys.ToArray(); + if (platforms.Length == 0) return 0; + + await SubscribeAndRecordAsync(platforms, cancellationToken).ConfigureAwait(false); + return platforms.Length; + } + + private async Task SubscribeAndRecordAsync(IReadOnlyList platforms, CancellationToken cancellationToken) + { + var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray(); + // PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence + // when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState + // changes are rare so a coarser interval is usually fine; deployments that need + // tighter health visibility can dial it down through GalaxyDriverOptions. + var results = await _subscriber.SubscribeBulkAsync( + probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false); + + var acquired = new (string Platform, int ItemHandle, string? Error)[platforms.Count]; + for (var i = 0; i < platforms.Count; i++) + { + var match = results.FirstOrDefault(r => string.Equals( + r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase)); + acquired[i] = (platforms[i], match is { WasSuccessful: true } ? match.ItemHandle : 0, + match?.ErrorMessage ?? ""); + } + + // Re-check the stop AFTER the await, atomically with the record + publish: a StopAsync that + // ran while SubscribeBulk was in flight already cleared the map and the registry entry, so + // recording now would resurrect the probe entry into the next runtime's registry. + bool stopped; + lock (_syncLock) + { + stopped = _disposed; + if (!stopped) + { + foreach (var (platform, itemHandle, _) in acquired) _itemHandlesByPlatform[platform] = itemHandle; + PublishBindingsLocked(); + } + } + + if (stopped) + { + var orphans = acquired.Where(a => a.ItemHandle > 0).Select(a => a.ItemHandle).ToArray(); + if (orphans.Length > 0) + { + try { await _subscriber.UnsubscribeBulkAsync(orphans, CancellationToken.None).ConfigureAwait(false); } + catch (Exception ex) + { + _logger.LogWarning(ex, + "PerPlatformProbeWatcher could not release {Count} probe handle(s) acquired after stop", orphans.Length); + } + } + return; + } + + foreach (var (platform, itemHandle, error) in acquired) + { + if (itemHandle <= 0) + { + _logger.LogWarning("PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", platform, error); + } + } + } + /// Snapshot of platform tag names currently watched. public IReadOnlyCollection WatchedPlatforms => [.. _itemHandlesByPlatform.Keys]; @@ -109,6 +234,8 @@ public sealed class PerPlatformProbeWatcher : IDisposable if (toRemove.Count > 0) { + // Drop the dead handles from the fan-out map before the (slow, fallible) unsubscribe. + PublishBindings(); try { await _subscriber.UnsubscribeBulkAsync( @@ -124,30 +251,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable if (toAdd.Count == 0) return; - var probeAddresses = toAdd.Select(p => p + ProbeSuffix).ToArray(); - // PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence - // when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState - // changes are rare so a coarser interval is usually fine; deployments that need - // tighter health visibility can dial it down through GalaxyDriverOptions. - var results = await _subscriber.SubscribeBulkAsync( - probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false); - - for (var i = 0; i < toAdd.Count; i++) - { - var platform = toAdd[i]; - var match = results.FirstOrDefault(r => string.Equals( - r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase)); - - var itemHandle = match is { WasSuccessful: true } ? match.ItemHandle : 0; - _itemHandlesByPlatform[platform] = itemHandle; - - if (itemHandle <= 0) - { - _logger.LogWarning( - "PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", - platform, match?.ErrorMessage ?? ""); - } - } + await SubscribeAndRecordAsync(toAdd, cancellationToken).ConfigureAwait(false); } /// @@ -196,20 +300,32 @@ public sealed class PerPlatformProbeWatcher : IDisposable }; } - /// Disposes the probe watcher and unsubscribes all tracked platforms. - public void Dispose() + /// + /// Stops the watcher and unsubscribes every tracked platform's probe handle on the gateway + /// (best-effort — a transport error is logged and swallowed). The async counterpart of + /// ; the driver's ShutdownAsync uses it so the unsubscribe + /// round-trip is awaited rather than blocked on. Idempotent. + /// + /// Bounds the unsubscribe round-trip. + /// A task that completes once the handles have been released (or the attempt gave up). + public async Task StopAsync(CancellationToken cancellationToken) { - if (_disposed) return; - _disposed = true; - - // Best-effort unsubscribe everything we know about. Run synchronously through - // GetAwaiter().GetResult() since Dispose is sync; transport errors are swallowed. - var liveHandles = _itemHandlesByPlatform.Values.Where(h => h > 0).ToArray(); - _itemHandlesByPlatform.Clear(); + int[] liveHandles; + lock (_syncLock) + { + if (_disposed) return; + _disposed = true; + liveHandles = [.. _itemHandlesByPlatform.Values.Where(h => h > 0)]; + _itemHandlesByPlatform.Clear(); + _registry?.Remove(SubscriptionRegistry.ProbeSubscriptionId); + } if (liveHandles.Length > 0) { - try { _subscriber.UnsubscribeBulkAsync(liveHandles, CancellationToken.None).GetAwaiter().GetResult(); } - catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher dispose unsubscribe failed"); } + try { await _subscriber.UnsubscribeBulkAsync(liveHandles, cancellationToken).ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher stop unsubscribe failed"); } } } + + /// Disposes the probe watcher and unsubscribes all tracked platforms (blocking on the round-trip). + public void Dispose() => StopAsync(CancellationToken.None).GetAwaiter().GetResult(); } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs index d6e4fb4b..7eb95234 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs @@ -187,9 +187,11 @@ internal sealed class EventPump : IAsyncDisposable NotifyStreamFault(cause); } } - catch (OperationCanceledException) when (ct.IsCancellationRequested) + catch (Exception) when (ct.IsCancellationRequested) { - // Clean shutdown — no log. + // Clean shutdown — no log. A cancelled gRPC stream may surface as RpcException(Cancelled) + // rather than OperationCanceledException; either way the pump was told to stop, so this + // is not a transport fault and must not wake the reconnect supervisor mid-teardown. } catch (Exception ex) { @@ -283,14 +285,19 @@ internal sealed class EventPump : IAsyncDisposable EventsDispatched.Add(1, _clientTag); } } - catch (OperationCanceledException) when (ct.IsCancellationRequested) + catch (Exception) when (ct.IsCancellationRequested) { // Clean shutdown. } catch (Exception ex) { + // The read loop is still draining the stream into a channel nobody reads: the pump is + // as dead as after a transport fault, but used to raise nothing — health stayed + // Healthy and the comms-loss watchdog never tripped (a clean termination evading the + // watchdog). Report it as a stream fault so the supervisor recreates the pump. _logger.LogWarning(ex, - "Galaxy EventPump dispatch loop ended with an exception — events past this point will be lost until restart."); + "Galaxy EventPump dispatch loop ended with an exception — notifying reconnect supervisor so the pump is recreated."); + NotifyStreamFault(ex); } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs index 0daae292..75550740 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyMxSession.cs @@ -60,6 +60,12 @@ public sealed class GalaxyMxSession : IAsyncDisposable /// internal Func? OpenAndRegisterOverrideForTests { get; set; } + /// + /// Test seam — replaces the gateway CloseSession round-trip in + /// so the bounded-close behaviour can be exercised against a never-answering fake. + /// + internal Func? CloseOverrideForTests { get; set; } + /// /// Connect the underlying gateway client + open an MXAccess session + register the /// configured client name. Idempotent — second calls are no-ops while a connect has @@ -96,7 +102,7 @@ public sealed class GalaxyMxSession : IAsyncDisposable { // A partial open (e.g. OpenSession succeeded but Register threw) must not leave a half-open // handle that blocks the next reconnect. Tear the partial state down so a retry rebuilds cleanly. - await TeardownSessionAsync().ConfigureAwait(false); + await TeardownSessionAsync(CancellationToken.None).ConfigureAwait(false); throw; } } @@ -112,10 +118,23 @@ public sealed class GalaxyMxSession : IAsyncDisposable public async Task RecreateAsync(MxGatewayClientOptions clientOptions, CancellationToken cancellationToken) { ObjectDisposedException.ThrowIf(_disposed, this); - await TeardownSessionAsync().ConfigureAwait(false); + await TeardownSessionAsync(cancellationToken).ConfigureAwait(false); await ConnectAsync(clientOptions, cancellationToken).ConfigureAwait(false); } + /// + /// Close the gateway session and release the owned client under + /// . The SDK's DisposeAsync closes with NO token, + /// i.e. under the 30 s DefaultCallTimeout — against a blackholed gateway that parks + /// the driver's ShutdownAsync (and the host's PostStop) for the full timeout. + /// A close that is cancelled or fails is NOT retried through the SDK dispose (which would + /// park again); the channel is dropped and the gateway reaps the session on its idle lease. + /// The session can be re-opened afterwards (this does not set _disposed). + /// + /// Bounds the gateway CloseSession round-trip. + /// A task that completes once the session and client are released. + public Task CloseAsync(CancellationToken cancellationToken) => TeardownSessionAsync(cancellationToken); + /// /// Test seam — attach a session opened externally (e.g. against an in-process gw /// fake). Skips the gateway-client construction so tests can drive the session @@ -143,24 +162,45 @@ public sealed class GalaxyMxSession : IAsyncDisposable { if (_disposed) return; _disposed = true; - await TeardownSessionAsync().ConfigureAwait(false); + await TeardownSessionAsync(CancellationToken.None).ConfigureAwait(false); } /// - /// Resets the connect flag and best-effort disposes + nulls the owned session and client. - /// Shared by (final teardown) and / - /// the partial-open recovery path (rebuild teardown) — it does NOT set _disposed, so - /// a torn-down session can be re-opened. + /// Resets the connect flag and best-effort closes + nulls the owned session and client. + /// Shared by (final teardown), (bounded + /// teardown) and / the partial-open recovery path (rebuild + /// teardown) — it does NOT set _disposed, so a torn-down session can be re-opened. /// - private async Task TeardownSessionAsync() + private async Task TeardownSessionAsync(CancellationToken cancellationToken) { + var wasConnected = _connected; _connected = false; + // The override models the gateway round-trip, so it runs only when there was a live + // session to close — a second teardown (CloseAsync then DisposeAsync) must not close twice. + var closeOverride = CloseOverrideForTests; + if (closeOverride is not null && wasConnected) + { + try { await closeOverride(cancellationToken).ConfigureAwait(false); } + catch (Exception ex) { _logger.LogWarning(ex, "GalaxyMxSession close failed or timed out (best-effort)"); } + } + if (_session is not null) { - try { await _session.DisposeAsync().ConfigureAwait(false); } - catch (Exception ex) { _logger.LogWarning(ex, "GalaxyMxSession session dispose failed (best-effort)"); } + var session = _session; _session = null; + try + { + // CloseAsync(ct) is the bounded gateway round-trip; DisposeAsync only re-runs an + // un-replied close (with no token), so it is reached only after a successful one. + await session.CloseAsync(cancellationToken).ConfigureAwait(false); + await session.DisposeAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "GalaxyMxSession close failed or timed out (best-effort) — dropping the channel; the gateway reaps the session on its idle lease."); + } } if (_ownedClient is not null) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs index a472f269..957459df 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs @@ -120,15 +120,87 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter // Bind the per-batch MX write operations behind the internal IMxWriteOps seam (see below) so the // whole write pipeline is unit-testable without the sealed SDK session (archreview 06/S-1). var ops = new SessionMxWriteOps(session, serverHandle); + return await WriteBatchAsync(ops, writes, securityResolver, cancellationToken).ConfigureAwait(false); + } + /// + /// Upper bound on write lanes in flight at once. A first-touch write is three round-trips + /// (AddItem → AdviseSupervisory → Write) and the batch used to run them strictly serially; + /// distinct tags are independent, so their lanes now overlap, bounded so a large batch cannot + /// flood the gateway's STA pump. Acks stay serial by design (see GalaxyDriver.AcknowledgeAsync). + /// + internal const int MaxConcurrentWriteLanes = 8; + + /// Test seam: drive the real batch pipeline against a fake . + /// The (fake) MX write operations. + /// The batch. + /// Per-tag classification resolver. + /// The cancellation token. + /// One result per request, in input order. + internal Task> WriteBatchForTestAsync( + IMxWriteOps ops, IReadOnlyList writes, + Func securityResolver, CancellationToken ct) + => WriteBatchAsync(ops, writes, securityResolver, ct); + + /// + /// Run a batch as one lane per distinct tag (OrdinalIgnoreCase): writes to the SAME tag + /// stay in input order inside their lane, lanes run concurrently up to + /// , and results land at their request's index. + /// A single-tag batch takes the direct path with no gate. + /// + private async Task> WriteBatchAsync( + IMxWriteOps ops, IReadOnlyList writes, + Func securityResolver, CancellationToken ct) + { var results = new WriteResult[writes.Count]; + if (writes.Count == 0) return results; + + // Group request indices by tag, first-seen order preserved so the lane start order is + // deterministic and a same-tag sequence keeps its authored order. + var lanes = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var laneOrder = new List>(); for (var i = 0; i < writes.Count; i++) { - results[i] = await WriteOneAsync(ops, writes[i], - securityResolver(writes[i].FullReference), cancellationToken) - .ConfigureAwait(false); + if (!lanes.TryGetValue(writes[i].FullReference, out var lane)) + { + lane = []; + lanes[writes[i].FullReference] = lane; + laneOrder.Add(lane); + } + lane.Add(i); } + + if (laneOrder.Count == 1) + { + await RunLaneAsync(laneOrder[0]).ConfigureAwait(false); + return results; + } + + using var gate = new SemaphoreSlim(MaxConcurrentWriteLanes, MaxConcurrentWriteLanes); + var tasks = new Task[laneOrder.Count]; + for (var l = 0; l < laneOrder.Count; l++) + { + var lane = laneOrder[l]; + tasks[l] = RunGatedAsync(lane); + } + await Task.WhenAll(tasks).ConfigureAwait(false); return results; + + async Task RunGatedAsync(List lane) + { + await gate.WaitAsync(ct).ConfigureAwait(false); + try { await RunLaneAsync(lane).ConfigureAwait(false); } + finally { gate.Release(); } + } + + async Task RunLaneAsync(List lane) + { + foreach (var i in lane) + { + results[i] = await WriteOneAsync(ops, writes[i], + securityResolver(writes[i].FullReference), ct).ConfigureAwait(false); + } + } } /// Test seam: drive the real single-write pipeline against a fake diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs index 15022173..7b6c6410 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs @@ -62,6 +62,16 @@ public sealed class ReconnectSupervisor : IDisposable private Task? _recoveryLoop; private CancellationTokenSource? _loopCts; private bool _disposed; + private int _cyclesStarted; + private int _cyclesReleased; + + /// Recovery cycles started so far (one per that left Healthy). Test seam. + internal int CyclesStarted => Volatile.Read(ref _cyclesStarted); + + /// Recovery cycles whose has been disposed. Equals + /// once every loop has exited — a cycle's CTS used to be released only + /// if it happened to be the LAST one when ran. Test seam. + internal int CyclesReleased => Volatile.Read(ref _cyclesReleased); /// Fires after every state transition. public event EventHandler? StateChanged; @@ -136,8 +146,13 @@ public sealed class ReconnectSupervisor : IDisposable TransitionLocked(State.TransportLost, cause.Message); - _loopCts = new CancellationTokenSource(); - _recoveryLoop = Task.Run(() => RecoveryLoopAsync(_loopCts.Token)); + // Each cycle owns its CTS and releases it when its loop exits (see RecoveryLoopAsync's + // finally). Before, every cycle allocated a fresh CTS and only the last one was ever + // disposed — one leaked CancellationTokenSource per outage for the driver's lifetime. + var cts = new CancellationTokenSource(); + _loopCts = cts; + Interlocked.Increment(ref _cyclesStarted); + _recoveryLoop = Task.Run(() => RecoveryLoopAsync(cts)); } } @@ -156,7 +171,26 @@ public sealed class ReconnectSupervisor : IDisposable } } - private async Task RecoveryLoopAsync(CancellationToken ct) + private async Task RecoveryLoopAsync(CancellationTokenSource cycleCts) + { + try + { + await RecoveryLoopCoreAsync(cycleCts.Token).ConfigureAwait(false); + } + finally + { + // Release this cycle's CTS once nothing can observe its token any more. Stop/Dispose may + // race us here: they Cancel (guarded against ODE) and Dispose (idempotent) the same object. + lock (_stateLock) + { + if (ReferenceEquals(_loopCts, cycleCts)) _loopCts = null; + } + cycleCts.Dispose(); + Interlocked.Increment(ref _cyclesReleased); + } + } + + private async Task RecoveryLoopCoreAsync(CancellationToken ct) { var attempt = 0; while (!ct.IsCancellationRequested) @@ -240,8 +274,13 @@ public sealed class ReconnectSupervisor : IDisposable } } - /// Disposes the supervisor and cancels any in-flight recovery loop. - public void Dispose() + /// + /// Cancels any in-flight recovery loop and waits for it to exit, without blocking the caller's + /// thread. The async counterpart of ; the driver's ShutdownAsync + /// uses it so a loop parked inside a reopen round-trip is awaited, not blocked on. Idempotent. + /// + /// A task that completes once the recovery loop has exited. + public async Task StopAsync() { if (_disposed) return; _disposed = true; @@ -250,13 +289,18 @@ public sealed class ReconnectSupervisor : IDisposable Task? loop; lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; } - cts?.Cancel(); + // The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS + // throws on Cancel, and a disposed one is a no-op on Dispose. + try { cts?.Cancel(); } catch (ObjectDisposedException) { /* loop already released it */ } if (loop is not null) { - try { loop.GetAwaiter().GetResult(); } catch { /* shutdown */ } + try { await loop.ConfigureAwait(false); } catch { /* shutdown */ } } cts?.Dispose(); } + + /// Disposes the supervisor and cancels any in-flight recovery loop (blocking until it exits). + public void Dispose() => StopAsync().GetAwaiter().GetResult(); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs index 512099f5..1a6ac2ff 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs @@ -18,10 +18,39 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; /// internal sealed class SubscriptionRegistry { + /// + /// Reserved subscription id for the per-platform ScanState probe handles + /// (PerPlatformProbeWatcher). Driver subscription ids are positive and + /// monotonic (), so a negative id can never collide. + /// The probe handles used to live only in the watcher's own map, so the + /// — which resolves subscribers through THIS registry — + /// dropped every probe event as "stale after unsubscribe" and per-platform host + /// status never updated. + /// + public const long ProbeSubscriptionId = -1; + private readonly ConcurrentDictionary _bySubscriptionId = new(); private readonly ConcurrentDictionary> _subscribersByItemHandle = new(); private long _nextSubscriptionId; + /// True when is currently registered. + /// The subscription identifier. + /// Whether the id is tracked. + public bool Contains(long subscriptionId) => _bySubscriptionId.ContainsKey(subscriptionId); + + /// + /// when the id is already tracked (so its stale reverse-map entries + /// are dropped), otherwise. Used by the probe watcher, whose single + /// reserved subscription is re-issued on every platform-set change. + /// + /// The subscription identifier. + /// The tag bindings for the subscription. + public void Upsert(long subscriptionId, IReadOnlyList bindings) + { + if (Contains(subscriptionId)) Rebind(subscriptionId, bindings); + else Register(subscriptionId, bindings); + } + /// Gets the number of tracked subscriptions. public int TrackedSubscriptionCount => _bySubscriptionId.Count; /// Gets the number of tracked item handles. @@ -100,6 +129,17 @@ internal sealed class SubscriptionRegistry } + /// + /// Drop every subscription and the whole reverse map. Used by the driver's runtime teardown: + /// once the gateway session is closed every item handle in here is dead, and the host + /// re-issues its subscriptions (fresh ids) after the next InitializeAsync. + /// + public void Clear() + { + _bySubscriptionId.Clear(); + _subscribersByItemHandle.Clear(); + } + /// Snapshot every active binding for diagnostic output. /// A flattened list of every tag binding across all tracked subscriptions. public IReadOnlyList SnapshotAllBindings() => diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor index 17468d7d..705f70c7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/GalaxyDriverForm.razor @@ -92,7 +92,7 @@
-
Default 5000 objects per page.
+
Objects per DiscoverHierarchy page the driver asks the gateway for. Default 5000.
@@ -138,7 +138,7 @@
-
Max 60. Used by Test Connect. Default 30.
+
Caps the Test Connect probe deadline (the probe's own budget still applies, so this only shortens it). Max 60. Default 30.
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index d569fd08..69e878db 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -28,6 +28,12 @@ public sealed class AdminOperationsActor : ReceiveActor private readonly IReadOnlyDictionary _probesByType; private readonly TagConfigValidationMode _tagConfigValidationMode; + /// + /// The coordinator's apply deadline. A Dispatching row with no expected-ack rows that is older + /// than this is a lost dispatch, closed by the in-flight check (fix round 1 / I7). + /// + private readonly TimeSpan _applyDeadline; + /// /// Where central→node commands are handed off (per-cluster mesh), or /// to publish on the mediator directly. @@ -36,6 +42,9 @@ public sealed class AdminOperationsActor : ReceiveActor private readonly ILoggingAdapter _log = Context.GetLogger(); + /// The StartDeployment whose compose is running off-actor, or null. See . + private StartDeployment? _composeInFlight; + /// Creates actor props for the admin operations actor. /// Factory for creating config database contexts. /// Reference to the deployment coordinator actor. @@ -45,15 +54,17 @@ public sealed class AdminOperationsActor : ReceiveActor /// Central comm actor commands are handed to, or to publish on the /// DistributedPubSub mediator directly (the behaviour before the per-cluster mesh). /// + /// The coordinator's apply deadline; defaults to (the coordinator's default source). /// Props configured to create an AdminOperationsActor. public static Props Props( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, - IActorRef? meshRouter = null) => + IActorRef? meshRouter = null, + TimeSpan? applyDeadline = null) => Akka.Actor.Props.Create(() => - new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter)); + new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter, applyDeadline)); /// Initializes a new instance of the AdminOperationsActor. /// Factory for creating config database contexts. @@ -61,27 +72,48 @@ public sealed class AdminOperationsActor : ReceiveActor /// Driver probes registered in DI; keyed by DriverType (case-insensitive). /// Deploy-gate TagConfig strictness mode (Warn default | Error opt-in). /// Central comm actor, or for the mediator. + /// The coordinator's apply deadline; defaults to (the coordinator's default source). public AdminOperationsActor( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, - IActorRef? meshRouter = null) + IActorRef? meshRouter = null, + TimeSpan? applyDeadline = null) { + // The same source the coordinator's default reads (ConfigPublishCoordinator.DefaultApplyDeadline IS + // DeployTimings.ApplyDeadline), so the lost-dispatch closure and the coordinator's own deadline agree. + _applyDeadline = applyDeadline ?? DeployTimings.ApplyDeadline; _dbFactory = dbFactory; _coordinator = coordinator; _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase); _tagConfigValidationMode = tagConfigValidationMode; _meshRouter = meshRouter; - ReceiveAsync(HandleStartDeploymentAsync); - ReceiveAsync(HandleTestDriverConnectAsync); + // Synchronous receives: the compose and the probe run off-actor and come back as messages, so + // the mailbox is never suspended for the length of a DB snapshot or a PLC connect timeout. + Receive(HandleStartDeployment); + Receive(HandleDeployComposed); + Receive(HandleDeployComposeFailed); + Receive(HandleTestDriverConnect); ReceiveAsync(HandleRestartDriverAsync); ReceiveAsync(HandleReconnectDriverAsync); Receive(HandleAcknowledgeAlarm); Receive(HandleShelveAlarm); } + /// + /// The reply text for an unexpected server-side failure (R03:4.1): a fixed sentence plus the request's + /// correlation id, never ex.Message — a SqlException or UriFormatException message carries + /// connection-string fragments and host names, and the reply reaches any ConfigEditor browser session. + /// The detail is in the central log under the same correlation id. + /// + /// The admin operation that failed. + /// The request's correlation token, as logged alongside the exception. + /// The operator-facing message. + internal static string ServerFault(string operation, object correlationId) => + $"{operation} failed on the server; see the central node log for correlation {correlationId}."; + /// /// Hands a central→node command to the mesh transport router, or publishes it directly when /// no router is wired. @@ -129,8 +161,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId}", msg.AlarmId); - replyTo.Tell(new AcknowledgeAlarmResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId} (correlation {CorrelationId})", msg.AlarmId, msg.CorrelationId); + replyTo.Tell(new AcknowledgeAlarmResult(false, ServerFault("Acknowledge", msg.CorrelationId), msg.CorrelationId)); } } @@ -176,215 +208,364 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId}", msg.Kind, msg.AlarmId); - replyTo.Tell(new ShelveAlarmResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId} (correlation {CorrelationId})", msg.Kind, msg.AlarmId, msg.CorrelationId); + replyTo.Tell(new ShelveAlarmResult(false, ServerFault("Shelve", msg.CorrelationId), msg.CorrelationId)); } } - private async Task HandleStartDeploymentAsync(StartDeployment msg) + /// + /// StartDeployment. The only serialised part is the actor-local + /// flag; the DB in-flight check, the snapshot read, validation, gzip and the row write all run on the + /// thread pool and come back as a / + /// message, so a slow or blocked ConfigDb never suspends the mailbox (R03:3.1; fix round 1 moved the + /// in-flight query off the actor thread too) and alarm acks keep flowing while it runs. + /// + /// + /// is what keeps the check correct once it is off-actor: it is set + /// BEFORE the off-actor work starts and cleared when the outcome message lands (both branches), so a + /// second request is refused in memory for the whole check-plus-compose and two artifacts can never + /// race for one coordinator. + /// + private void HandleStartDeployment(StartDeployment msg) { var replyTo = Sender; + if (_composeInFlight is { } inFlightRequest) + { + replyTo.Tell(new StartDeploymentResult( + StartDeploymentOutcome.AnotherDeploymentInFlight, + DeploymentId: null, + RevisionHash: null, + Message: $"A deployment requested by {inFlightRequest.CreatedBy} is still being composed.", + msg.CorrelationId)); + return; + } + + _composeInFlight = msg; + // Captured here: Context/Sender are not available on the thread-pool thread. + var sourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown"; + var self = Self; + Task.Run(async () => + { + var refusal = await CheckInFlightAsync(msg).ConfigureAwait(false); + return refusal ?? await ComposeDeploymentAsync(msg, sourceNode).ConfigureAwait(false); + }) + .PipeTo(self, + success: composed => new DeployComposed(msg, replyTo, composed.Result, composed.Dispatch), + failure: ex => new DeployComposeFailed(msg, replyTo, ex)); + } + + /// + /// The DB half of the in-flight check, off the actor thread. Refuses while a deployment is in flight — + /// keeping the coordinator's state unambiguous — EXCEPT a lost dispatch, which it closes first. + /// + /// The request. + /// The refusal, or null to proceed. + /// + /// + /// Lost dispatch (fix round 1 / I7). A Dispatching row with ZERO + /// NodeDeploymentState rows was never taken by the coordinator (its HandleDispatch + /// seeds the expected set and flips the status in one SaveChanges). The coordinator's PreStart + /// recovers such a row — but only one that already existed when it started. A row committed AFTER + /// PreStart whose Tell never arrived (this actor died between the compose's commit and + /// ) has no owner at all, and refused every later deploy forever. + /// Once it is past the apply deadline nothing can still be legitimately dispatching it, so it is + /// closed TimedOut with a reason and the new deploy proceeds. A late DispatchDeployment + /// for it is ignored by the coordinator's terminal-state guard. + /// + /// + /// One indexed projection (IX_Deployment_Status) plus, only for a Dispatching row past the + /// deadline, a count and a tracked update. + /// + /// + private async Task CheckInFlightAsync(StartDeployment msg) + { + await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false); + var inflight = await db.Deployments + .AsNoTracking() + .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) + .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, d.Status, d.CreatedAtUtc }) + .ToListAsync() + .ConfigureAwait(false); + + foreach (var row in inflight) + { + if (row.Status == DeploymentStatus.Dispatching + && DateTime.UtcNow - row.CreatedAtUtc > _applyDeadline + && !await db.NodeDeploymentStates.AnyAsync(s => s.DeploymentId == row.DeploymentId).ConfigureAwait(false) + && await CloseLostDispatchAsync(db, row.DeploymentId).ConfigureAwait(false)) + { + continue; + } + + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.AnotherDeploymentInFlight, + DeploymentId: new DeploymentId(row.DeploymentId), + RevisionHash: null, + Message: $"Deployment {row.DeploymentId:N} is still in flight.", + msg.CorrelationId), + Dispatch: null); + } + return null; + } + + /// Closes a lost dispatch as TimedOut, unless it moved on meanwhile (optimistic, on its RowVersion). + /// The context. + /// The row. + /// when the row is now closed. + private async Task CloseLostDispatchAsync(OtOpcUaConfigDbContext db, Guid deploymentId) + { + var stamp = await db.Deployments + .AsNoTracking() + .Where(d => d.DeploymentId == deploymentId && d.Status == DeploymentStatus.Dispatching) + .Select(d => new { d.RowVersion }) + .FirstOrDefaultAsync() + .ConfigureAwait(false); + if (stamp is null) return true; // no longer Dispatching — not in flight as far as this check goes + + // Stub update: the row's ArtifactBlob is never read just to change its status. + var stub = new Deployment + { + DeploymentId = deploymentId, RevisionHash = string.Empty, CreatedBy = string.Empty, + RowVersion = stamp.RowVersion, + Status = DeploymentStatus.TimedOut, + FailureReason = + "Dispatch was lost before any node was expected to ack (the coordinator was never told, or never " + + "took it) and the apply deadline has passed; closed by the next StartDeployment. Redeploy.", + }; + var entry = db.Deployments.Attach(stub); + entry.Property(d => d.Status).IsModified = true; + entry.Property(d => d.FailureReason).IsModified = true; try { - await using var db = await _dbFactory.CreateDbContextAsync(); + await db.SaveChangesAsync().ConfigureAwait(false); + } + catch (DbUpdateConcurrencyException) + { + // The coordinator took it after all (status flipped, RowVersion moved): it is genuinely in flight. + return false; + } + _log.Warning( + "StartDeployment: closed lost dispatch {DeploymentId} as TimedOut — Dispatching with no expected-ack rows " + + "past the {Deadline} apply deadline (the coordinator never took it)", deploymentId, _applyDeadline); + return true; + } - // Refuse if any deployment is already in flight — keeps the coordinator's state - // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. - var inflight = await db.Deployments - .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) - .Select(d => d.DeploymentId) - .FirstOrDefaultAsync(); - if (inflight != Guid.Empty) - { - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.AnotherDeploymentInFlight, - DeploymentId: new DeploymentId(inflight), - RevisionHash: null, - Message: $"Deployment {inflight:N} is still in flight.", - msg.CorrelationId)); - return; - } + private void HandleDeployComposed(DeployComposed msg) + { + _composeInFlight = null; + // The row is committed (the compose task completed), so the coordinator can be told now; it reads + // the row back by id when it seeds the expected-ack set. + if (msg.Dispatch is { } dispatch) _coordinator.Tell(dispatch); + msg.ReplyTo.Tell(msg.Result); + } - // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject - // on ANY validation error. The config is now canonical — the company overlay loader emits - // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId - // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions, - // …) gates the deploy. A green build in this repo does not prove the config is valid; this - // is the last guard before a bad address space (or a non-derived EquipmentId) ships. - // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can - // (rarely) be a false positive if an external-id reservation release has not yet been - // committed/visible when the snapshot is read — operators seeing a spurious one should - // check ExternalIdReservation state before re-submitting. - var draft = await DraftSnapshotFactory.FromConfigDbAsync(db); - var errors = DraftValidator.Validate(draft).ToList(); + private void HandleDeployComposeFailed(DeployComposeFailed msg) + { + _composeInFlight = null; + _log.Error(msg.Error, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.Request.CreatedBy, msg.Request.CorrelationId); + msg.ReplyTo.Tell(new StartDeploymentResult( + StartDeploymentOutcome.Rejected, + DeploymentId: null, + RevisionHash: null, + Message: ServerFault("StartDeployment", msg.Request.CorrelationId), + msg.Request.CorrelationId)); + } - // Cluster-topology guard. The SQL - // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode) - // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an - // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at - // Hot/Warm and the constraint stays green, which would boot the runtime into an - // InvalidTopology band. ValidateClusterTopology catches that drift, but it isn't carried on - // the generation-versioned DraftSnapshot (the cluster/node rows aren't versioned), so it must - // be run separately here against the live rows. Read-only (AsNoTracking); errors fold into the - // same reject summary alongside the snapshot rules so a deploy failing either check is - // rejected with both sets of messages. ClusterId-ordered for a deterministic summary. - var clusters = await db.ServerClusters - .AsNoTracking() - .OrderBy(c => c.ClusterId) - .ToListAsync(); - var nodesByCluster = (await db.ClusterNodes - .AsNoTracking() - .ToListAsync()) - .GroupBy(n => n.ClusterId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); - foreach (var cluster in clusters) - { - var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns) - ? (IReadOnlyList)ns - : []; - errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes)); - } + /// Outcome of the off-actor compose: the reply to send, and the dispatch to issue when Accepted. + private sealed record ComposeOutcome(StartDeploymentResult Result, DispatchDeployment? Dispatch); - // Deploy-time TagConfig strictness inspection over equipment tags. The - // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would - // REJECT — a present-but-invalid enum value, an absent or malformed address field, a - // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads - // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum; - // it now rejects the tag, so these warnings describe rejections.) In Warn mode - // (default) these are non-blocking (logged + appended to the accepted result Message); in Error - // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed. - // Running servers are untouched either way — the gate only sees re-deploys. - // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId / - // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type. - var driverTypeById = draft.DriverInstances - .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal); - var driverByDevice = draft.Devices - .GroupBy(d => d.DeviceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal); - var deviceConfigById = draft.Devices - .GroupBy(d => d.DeviceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal); - var tagConfigWarnings = new List<(string TagId, string Text)>(); - foreach (var t in draft.Tags) - { - if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue; - if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue; - foreach (var w in EquipmentTagConfigInspector.Inspect( - driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId))) - tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}")); - } - if (tagConfigWarnings.Count > 0) - { - if (_tagConfigValidationMode == TagConfigValidationMode.Error) - errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId))); - else - foreach (var x in tagConfigWarnings) - _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text); - } + private sealed record DeployComposed( + StartDeployment Request, IActorRef ReplyTo, StartDeploymentResult Result, DispatchDeployment? Dispatch); - if (errors.Count > 0) - { - var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}")); - _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}", - errors.Count, summary); - replyTo.Tell(new StartDeploymentResult( + private sealed record DeployComposeFailed(StartDeployment Request, IActorRef ReplyTo, Exception Error); + + /// + /// The deploy compose, off the actor thread. Touches no actor state: everything it needs is in + /// , and the readonly fields captured at + /// construction (, , — + /// Akka's logging adapter is thread-safe). + /// + private async Task ComposeDeploymentAsync(StartDeployment msg, string sourceNode) + { + await using var db = await _dbFactory.CreateDbContextAsync(); + + // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject + // on ANY validation error. The config is now canonical — the company overlay loader emits + // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId + // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions, + // …) gates the deploy. A green build in this repo does not prove the config is valid; this + // is the last guard before a bad address space (or a non-derived EquipmentId) ships. + // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can + // (rarely) be a false positive if an external-id reservation release has not yet been + // committed/visible when the snapshot is read — operators seeing a spurious one should + // check ExternalIdReservation state before re-submitting. + var draft = await DraftSnapshotFactory.FromConfigDbAsync(db); + var errors = DraftValidator.Validate(draft).ToList(); + + // Cluster-topology guard. The SQL + // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode) + // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an + // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at + // Hot/Warm and the constraint stays green, which would boot the runtime into an + // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of + // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows. + // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing + // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary. + // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT + // transaction), not a second query that could observe a later edit. + var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList(); + var nodesByCluster = draft.ClusterNodes + .GroupBy(n => n.ClusterId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); + foreach (var cluster in clusters) + { + var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns) + ? (IReadOnlyList)ns + : []; + errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes)); + } + + // Deploy-time TagConfig strictness inspection over equipment tags. The + // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would + // REJECT — a present-but-invalid enum value, an absent or malformed address field, a + // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads + // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum; + // it now rejects the tag, so these warnings describe rejections.) In Warn mode + // (default) these are non-blocking (logged + appended to the accepted result Message); in Error + // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed. + // Running servers are untouched either way — the gate only sees re-deploys. + // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId / + // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type. + var driverTypeById = draft.DriverInstances + .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal); + var driverByDevice = draft.Devices + .GroupBy(d => d.DeviceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal); + var deviceConfigById = draft.Devices + .GroupBy(d => d.DeviceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal); + var tagConfigWarnings = new List<(string TagId, string Text)>(); + foreach (var t in draft.Tags) + { + if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue; + if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue; + foreach (var w in EquipmentTagConfigInspector.Inspect( + driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId))) + tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}")); + } + if (tagConfigWarnings.Count > 0) + { + if (_tagConfigValidationMode == TagConfigValidationMode.Error) + errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId))); + else + foreach (var x in tagConfigWarnings) + _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text); + } + + if (errors.Count > 0) + { + var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}")); + _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}", + errors.Count, summary); + return new ComposeOutcome( + new StartDeploymentResult( StartDeploymentOutcome.Rejected, DeploymentId: null, RevisionHash: null, Message: summary, - msg.CorrelationId)); - return; + msg.CorrelationId), + Dispatch: null); + } + + // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough) + // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc + // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct + // sources are counted (the compile cache keys on source, so duplicates collapse to one unit). + // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard + // reject; this advisory rides in the Accepted Message so the UI can show it. + // advisory is declared outside the inner try so the Accepted reply below can still read it even + // when the guardrail throws. A failed estimate must NEVER block a valid deploy — the caller's + // failure branch is reserved for genuine seal/save failures. + string? advisory = null; + try + { + const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) + // Projected from the snapshot already in hand — Script.SourceCode used to be read three times. + var compiled = draft.Scripts + .Select(s => s.SourceCode) + .Where(src => !PassthroughScript.TryMatch(src, out _)) + .Distinct(StringComparer.Ordinal) + .Count(); + if (compiled > 0) + { + var estMiB = compiled * PerScriptMiB; + _log.Warning( + "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it", + compiled, estMiB); + advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)"; } + } + catch (Exception ex) + { + // Guardrail is advisory-only — a failed estimate must never block a valid deploy. + _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped"); + advisory = null; + } - // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough) - // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc - // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct - // sources are counted (the compile cache keys on source, so duplicates collapse to one unit). - // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard - // reject; this advisory rides in the Accepted Message so the UI can show it. - // advisory is declared outside the inner try so the Accepted reply below can still read it even - // when the guardrail query throws (e.g. transient SQL). A failed estimate must NEVER block a - // valid deploy — the outer catch is reserved for genuine seal/save failures. - string? advisory = null; - try - { - const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) - var scriptSources = await db.Scripts.AsNoTracking().Select(s => s.SourceCode).ToListAsync(); - var compiled = scriptSources - .Where(src => !PassthroughScript.TryMatch(src, out _)) - .Distinct(StringComparer.Ordinal) - .Count(); - if (compiled > 0) - { - var estMiB = compiled * PerScriptMiB; - _log.Warning( - "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it", - compiled, estMiB); - advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)"; - } - } - catch (Exception ex) - { - // Guardrail is advisory-only — a failed estimate must never block a valid deploy. - _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped"); - advisory = null; - } + // Composed from the validated snapshot, never from a second read: an edit that lands between + // validation and here is simply not in this artifact (R03:3.2). + var artifact = ConfigComposer.Compose(draft); + var deploymentId = DeploymentId.NewId(); + var revHash = RevisionHash.Parse(artifact.RevisionHash); - var artifact = await ConfigComposer.SnapshotAndFlattenAsync(db); - var deploymentId = DeploymentId.NewId(); - var revHash = RevisionHash.Parse(artifact.RevisionHash); + db.Deployments.Add(new Deployment + { + DeploymentId = deploymentId.Value, + RevisionHash = artifact.RevisionHash, + Status = DeploymentStatus.Dispatching, + CreatedBy = msg.CreatedBy, + ArtifactBlob = artifact.Blob, + }); - db.Deployments.Add(new Deployment - { - DeploymentId = deploymentId.Value, - RevisionHash = artifact.RevisionHash, - Status = DeploymentStatus.Dispatching, - CreatedBy = msg.CreatedBy, - ArtifactBlob = artifact.Blob, - }); + // Marker ConfigEdit row so the audit timeline shows the deployment snapshot. + db.ConfigEdits.Add(new ConfigEdit + { + EntityType = "Deployment", + EntityId = deploymentId.Value, + FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}", + EditedBy = msg.CreatedBy, + SourceNode = sourceNode, + }); - // Marker ConfigEdit row so the audit timeline shows the deployment snapshot. - db.ConfigEdits.Add(new ConfigEdit - { - EntityType = "Deployment", - EntityId = deploymentId.Value, - FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}", - EditedBy = msg.CreatedBy, - SourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown", - }); + await db.SaveChangesAsync(); - await db.SaveChangesAsync(); + // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so + // the operator sees the non-blocking strictness findings on a successful deploy. + var messageParts = new List(); + if (advisory is not null) messageParts.Add(advisory); + messageParts.AddRange(tagConfigWarnings.Select(x => x.Text)); + var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null; - _coordinator.Tell(new DispatchDeployment(deploymentId, revHash, msg.CorrelationId)); - - // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so - // the operator sees the non-blocking strictness findings on a successful deploy. - var messageParts = new List(); - if (advisory is not null) messageParts.Add(advisory); - messageParts.AddRange(tagConfigWarnings.Select(x => x.Text)); - var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null; - - replyTo.Tell(new StartDeploymentResult( + return new ComposeOutcome( + new StartDeploymentResult( StartDeploymentOutcome.Accepted, deploymentId, revHash, Message: acceptedMessage, - msg.CorrelationId)); - } - catch (Exception ex) - { - _log.Error(ex, "StartDeployment failed for {CreatedBy}", msg.CreatedBy); - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Rejected, - DeploymentId: null, - RevisionHash: null, - Message: ex.Message, - msg.CorrelationId)); - } + msg.CorrelationId), + new DispatchDeployment(deploymentId, revHash, msg.CorrelationId)); } - private async Task HandleTestDriverConnectAsync(TestDriverConnect msg) + /// + /// Test Connect runs on the thread pool and pipes its reply straight to the requester: a probe + /// against a black-holed PLC (60 s clamp) must not suspend the singleton's mailbox (R03:3.1). Every + /// path inside the task yields a , so the requester never + /// sees a Status.Failure. + /// + private void HandleTestDriverConnect(TestDriverConnect msg) { var replyTo = Sender; if (!_probesByType.TryGetValue(msg.DriverType, out var probe)) @@ -397,6 +578,11 @@ public sealed class AdminOperationsActor : ReceiveActor return; } + Task.Run(() => ProbeAsync(probe, msg)).PipeTo(replyTo); + } + + private async Task ProbeAsync(IDriverProbe probe, TestDriverConnect msg) + { var clampedSec = Math.Clamp(msg.TimeoutSeconds, 1, 60); var timeout = TimeSpan.FromSeconds(clampedSec); using var cts = new CancellationTokenSource(timeout); @@ -406,28 +592,28 @@ public sealed class AdminOperationsActor : ReceiveActor { var result = await probe.ProbeAsync(msg.ConfigJson, timeout, cts.Token); sw.Stop(); - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( result.Ok, result.Message, result.Ok ? sw.Elapsed.TotalMilliseconds : (double?)null, - msg.CorrelationId)); + msg.CorrelationId); } catch (OperationCanceledException) { - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( false, $"Probe timed out after {clampedSec}s.", null, - msg.CorrelationId)); + msg.CorrelationId); } catch (Exception ex) { - _log.Error(ex, "Probe for {DriverType} threw", msg.DriverType); - replyTo.Tell(new TestDriverConnectResult( + _log.Error(ex, "Probe for {DriverType} threw (correlation {CorrelationId})", msg.DriverType, msg.CorrelationId); + return new TestDriverConnectResult( false, - ex.Message, + ServerFault("Test Connect", msg.CorrelationId), null, - msg.CorrelationId)); + msg.CorrelationId); } } @@ -457,8 +643,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId}", msg.DriverInstanceId); - replyTo.Tell(new RestartDriverResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId); + replyTo.Tell(new RestartDriverResult(false, ServerFault("RestartDriver", msg.CorrelationId), msg.CorrelationId)); } } @@ -487,8 +673,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId}", msg.DriverInstanceId); - replyTo.Tell(new ReconnectDriverResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId); + replyTo.Tell(new ReconnectDriverResult(false, ServerFault("ReconnectDriver", msg.CorrelationId), msg.CorrelationId)); } } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs index d73dd7b3..39735ddb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs @@ -1,17 +1,17 @@ using System.IO.Compression; using System.Security.Cryptography; using System.Text.Json; -using Microsoft.EntityFrameworkCore; using ZB.MOM.WW.OtOpcUa.Configuration; -using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; /// /// Pure snapshot composer: reads the current live-edit state from /// and serialises it into a deterministic, gzip-compressed byte[] artifact + SHA-256 hex revision hash. -/// Determinism comes from sorting every collection by its natural key before serialising, so two snapshots -/// over the same DB state always produce the same hash regardless of EF row ordering; compression wraps the +/// Determinism comes from reading every collection ordered by its natural key on the server +/// (), so two snapshots over the same DB state always produce the same hash +/// — and the same hash as the pre-R03:3.2 composer, which used those same queries; compression wraps the /// final bytes and reorders nothing. The hash is taken over the COMPRESSED bytes — see . /// public static class ConfigComposer @@ -25,45 +25,71 @@ public static class ConfigComposer }; /// Reads the current configuration and returns a deterministic snapshot blob with revision hash. + /// + /// One consistent read (, a SNAPSHOT transaction on + /// SQL Server) handed to . The deploy path does not call this: it validates and + /// composes from the one it already holds, so nothing can land between the + /// validator's read and the composer's. This overload serves the drift probe and tests. + /// /// The configuration database context. /// The cancellation token for the operation. /// A deterministic containing the serialised snapshot bytes and its revision hash. public static async Task SnapshotAndFlattenAsync( OtOpcUaConfigDbContext db, CancellationToken ct = default) + => Compose(await DraftSnapshotFactory.FromConfigDbAsync(db, ct).ConfigureAwait(false)); + + /// + /// Serialises an already-read into the deterministic artifact. Pure over + /// its input: the same snapshot always yields the same bytes and hash, whatever the DB holds by now. + /// + /// + /// Serialises the collections IN THE ORDER GIVEN. Determinism comes from + /// , which reads every table ordered by its natural key + /// on the server; a snapshot built any other way must be ordered the same way or its hash will differ. + /// + /// The rows to compose — the SAME rows the validator was run over. + /// The compressed artifact blob and its revision hash. + public static ConfigArtifact Compose(DraftSnapshot snapshot) { + ArgumentNullException.ThrowIfNull(snapshot); + // v3 greenfield snapshot: the Raw tree (RawFolders → DriverInstances(+RawFolderId) → Devices(+DeviceConfig) // → TagGroups → Tags(DeviceId/TagGroupId, no EquipmentId/FolderPath/DriverInstanceId)) plus the UNS // projection (UnsAreas/UnsLines/Equipment/UnsTagReferences) and the retained VirtualTags/ScriptedAlarms/ // PollGroups/NodeAcls/Scripts. The Namespaces snapshot is RETIRED (the two OPC UA namespaces are implicit). // RevisionHash inputs follow the blob automatically — adding/removing a table shifts the SHA-256. - var snapshot = new + // NO re-sort here (fix round 1): the snapshot's collections arrive ordered by their natural key from + // DraftSnapshotFactory's server-side ORDER BY — the same order, and so the same bytes and RevisionHash, + // as the pre-R03:3.2 composer's own queries. An in-memory ordinal re-sort orders mixed-case keys + // differently from the SQL collation and silently re-hashed every unchanged configuration. + var ordered = new { - Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), - Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), - RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), - DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), - Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), - TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), - Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), - Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), - UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), - PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), - UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), - UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), - NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + Clusters = snapshot.ServerClusters, + Nodes = snapshot.ClusterNodes, + RawFolders = snapshot.RawFolders, + DriverInstances = snapshot.DriverInstances, + Devices = snapshot.Devices, + TagGroups = snapshot.TagGroups, + Equipment = snapshot.Equipment, + Tags = snapshot.Tags, + UnsTagReferences = snapshot.UnsTagReferences, + PollGroups = snapshot.PollGroups, + UnsAreas = snapshot.UnsAreas, + UnsLines = snapshot.UnsLines, + NodeAcls = snapshot.NodeAcls, // the data-plane credentials travel with the config, because a driver-only node // has no ConfigDb to read them from (per-cluster mesh Phase 4) and would otherwise deny every // UserName token. Only the peppered HMAC of each secret ships — never the secret — and the // pepper is node configuration that is deliberately NOT in the artifact, so a stolen blob // still cannot be turned into a working credential. This is also why revoking a key reaches a // driver-only node only on the next deployment; see the ApiKey entity's remarks. - ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), - Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), - VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), - ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + ApiKeys = snapshot.ApiKeys, + Scripts = snapshot.Scripts, + VirtualTags = snapshot.VirtualTags, + ScriptedAlarms = snapshot.ScriptedAlarms, }; - return Pack(JsonSerializer.SerializeToUtf8Bytes(snapshot, JsonOptions)); + return Pack(JsonSerializer.SerializeToUtf8Bytes(ordered, JsonOptions)); } /// diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs index 2ca3026c..3f0aa914 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs @@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; /// Dedup is two-layer: in-buffer (the below collapses /// duplicate EventIds before flush) and at the database via the filtered unique index /// UX_ConfigAuditLog_EventId (cross-restart safety — a retry of an already-flushed -/// batch hits the constraint and we drop the duplicate insert without losing the rest of -/// the batch). +/// batch hits the constraint; drops exactly the already-persisted EventIds +/// and re-saves the rest, so the duplicate is dropped without losing the rest of the batch). /// /// Implements the shared seam: is a /// best-effort, never-throwing entry point that simply Tells this actor and returns @@ -92,23 +92,8 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter try { - using var db = _dbFactory.CreateDbContext(); - foreach (var evt in snapshot) - { - db.ConfigAuditLogs.Add(new ConfigAuditLog - { - Timestamp = evt.OccurredAtUtc.UtcDateTime, - Principal = evt.Actor, - EventType = $"{evt.Category}:{evt.Action}", - NodeId = evt.SourceNode, - DetailsJson = evt.DetailsJson, - EventId = evt.EventId, - CorrelationId = evt.CorrelationId, - Outcome = evt.Outcome.ToString(), - }); - } - db.SaveChanges(); - _log.Debug("AuditWriter flushed {Count} events", snapshot.Count); + var dropped = SaveBatch(snapshot); + _log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped); } catch (Exception ex) { @@ -116,6 +101,102 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter } } + private static ConfigAuditLog ToRow(AuditEvent evt) => new() + { + Timestamp = evt.OccurredAtUtc.UtcDateTime, + Principal = evt.Actor, + EventType = $"{evt.Category}:{evt.Action}", + NodeId = evt.SourceNode, + DetailsJson = evt.DetailsJson, + EventId = evt.EventId, + CorrelationId = evt.CorrelationId, + Outcome = evt.Outcome.ToString(), + }; + + /// + /// Persists one flush, dropping ONLY the rows the database will not take (R03:2.6, fix round 1 / I4). + /// + /// The flush. + /// How many rows were dropped. + /// + /// + /// Why not detach ex.Entries. SQL Server inserts the batch as one statement batch, so + /// the raised for ONE duplicate EventId names EVERY entry in + /// the batch. Detaching what it names dropped the whole flush — the exact loss the fix was for + /// (pinned against a real SQL Server by On_sql_server_a_duplicate_event_id_costs_only_that_row). + /// + /// + /// So, on a rejection that names entries (a constraint violation; 2601/2627 for the unique index): + /// (1) ask the database which of the batch's EventIds it already holds and drop exactly those — + /// the at-least-once re-delivery the dedup contract expects; (2) re-save the rest as a batch; (3) if + /// that is still rejected (some OTHER per-row constraint), fall back to row-by-row and drop only the + /// rows that fail on their own. Every attempt uses a fresh context and fresh entities, so no state + /// from a failed save leaks into the next one. A failure that names no entry (connection loss, + /// deadlock) propagates — the whole-batch failure it always was. Bounded: at most 2 + N saves. + /// + /// + private int SaveBatch(IReadOnlyList batch) + { + if (TrySave(batch, out var first)) return 0; + _log.Debug(first, "AuditWriter: a batch of {Count} was rejected; dropping only already-persisted EventIds", batch.Count); + + HashSet existing; + using (var db = _dbFactory.CreateDbContext()) + { + var ids = batch.Select(e => (Guid?)e.EventId).ToList(); + existing = db.ConfigAuditLogs + .AsNoTracking() + .Where(r => ids.Contains(r.EventId)) + .Select(r => r.EventId!.Value) + .ToHashSet(); + } + foreach (var id in existing) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", id); + + var remaining = batch.Where(e => !existing.Contains(e.EventId)).ToList(); + var dropped = batch.Count - remaining.Count; + if (remaining.Count == 0 || TrySave(remaining, out _)) return dropped; + + // Still rejected with no duplicate left to blame: isolate the offending row(s). + foreach (var evt in remaining) + { + if (TrySave([evt], out var rowError)) continue; + dropped++; + if (IsUniqueViolation(rowError!)) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", evt.EventId); + else + _log.Warning(rowError, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", evt.EventId); + } + return dropped; + } + + /// Saves on a fresh context; a rejection that names entries is reported, anything else throws. + /// The rows to save. + /// The row-level rejection, when this returns false. + /// when saved. + private bool TrySave(IReadOnlyList events, out DbUpdateException? rejection) + { + using var db = _dbFactory.CreateDbContext(); + foreach (var evt in events) db.ConfigAuditLogs.Add(ToRow(evt)); + try + { + db.SaveChanges(); + rejection = null; + return true; + } + catch (DbUpdateException ex) when (ex.Entries.Count > 0 || IsUniqueViolation(ex)) + { + rejection = ex; + return false; + } + } + + /// SQL Server 2601 (unique index) / 2627 (unique constraint), the dedup contract's expected rejection. + /// The update failure. + /// for a duplicate-key rejection. + private static bool IsUniqueViolation(DbUpdateException ex) => + ex.InnerException is Microsoft.Data.SqlClient.SqlException { Number: 2601 or 2627 }; + /// protected override void PreRestart(Exception reason, object message) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs index cb1ea253..16f7b5d4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs @@ -201,6 +201,9 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers LoadContactsFromDb(); }); Receive(HandleContactsLoaded); + Receive(msg => _log.Debug("Mesh PSK refresh completed for {Count} cluster(s)", msg.ClusterCount)); + Receive(msg => _log.Warning(msg.Error, + "Mesh PSK refresh failed; previously resolved keys stay in service until the next refresh")); Receive(HandleCommandDispatched); // A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler @@ -710,6 +713,20 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers _targets = msg.Targets; + // R03:1.3 — release the channels of endpoints that left the dispatch set (host renamed, port + // moved). Only the resolved dialer is asked; nothing is resolved just to prune it. + if (_dialer is { } dialer) + { + try + { + dialer.RetainOnly(_targets.Select(t => t.Endpoint).ToHashSet(StringComparer.Ordinal)); + } + catch (Exception ex) + { + _log.Warning(ex, "Mesh dialer failed to release channels for retired endpoints; they stay cached until the next refresh"); + } + } + if (_targets.Count == 0) { _log.Warning( @@ -780,7 +797,37 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers } pskProvider.SetKnownClusters(msg.ClusterIds); - pskProvider.InvalidateAll(); + + // R03:3.4 — refresh the per-cluster keys OFF the request path and swap them in atomically, + // instead of dropping the cache and making the first ack per cluster re-read the store on a + // gRPC thread. A provider that does not pre-resolve (the interface default, and every fake) + // still drops its cache here, synchronously, exactly as before. + CancellationToken ct; + try + { + ct = _lifecycle.Token; + } + catch (ObjectDisposedException) + { + return; // Stopping. + } + var self = Self; + Task refresh; + try + { + refresh = pskProvider.RefreshAsync(msg.ClusterIds, ct); + } + catch (Exception ex) + { + _log.Warning(ex, "Mesh PSK refresh threw synchronously; cached keys are unchanged until the next refresh"); + refresh = Task.CompletedTask; + } + if (!refresh.IsCompletedSuccessfully) + { + refresh.PipeTo(self, + success: () => new PskRefreshed(msg.ClusterIds.Count), + failure: ex => new PskRefreshFailed(ex)); + } if (!_gatePublished) { @@ -795,6 +842,14 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// Internal tick asking for a contact-point refresh. public sealed record RefreshContacts; + /// The asynchronous PSK refresh (R03:3.4) completed. + /// How many clusters were refreshed. + public sealed record PskRefreshed(int ClusterCount); + + /// The asynchronous PSK refresh (R03:3.4) threw; the previous keys stay in service. + /// The failure. + public sealed record PskRefreshFailed(Exception Error); + /// Result of a contact-point load, piped back from the DB read. /// /// The gRPC dispatch set: one entry per enabled, non-maintenance row that carries a diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs index a7282694..bad1956c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs @@ -106,6 +106,27 @@ public interface IMeshPskProvider /// Drops every cached key. void InvalidateAll(); + + /// + /// Re-reads the keys for from the store OFF the request path and + /// swaps them into the cache atomically (R03:3.4). Until the refresh completes, lookups keep + /// serving the previous key; a cluster whose read FAILS keeps its previous key too, while a cluster whose + /// secret the store reports ABSENT is revoked (no key from then on). Replaces + /// "invalidate on every refresh", which made the first inbound ack per cluster per + /// ContactRefreshSeconds block a gRPC thread on a synchronous, 3 s-bounded store read. + /// + /// The clusters central currently knows (the same set given to ). + /// Cancels the refresh. + /// Completes when the swap has happened. + /// + /// The default implementation is the pre-R03:3.4 behaviour — drop the cache and let the next lookup + /// re-read — so a fake or a non-caching provider needs nothing more. + /// + Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) + { + InvalidateAll(); + return Task.CompletedTask; + } } /// @@ -144,6 +165,16 @@ public sealed class MeshPskProvider : IMeshPskProvider private readonly ConcurrentDictionary> _cache = new(StringComparer.OrdinalIgnoreCase); + /// + /// Keys pre-resolved by , consulted after the config map and before the + /// lazy . Swapped wholesale (volatile reference to an immutable dictionary), + /// never mutated, so a lookup racing a refresh sees the whole previous snapshot or the whole next + /// one. A cluster missing from it — never refreshed, or whose read failed — falls through to the + /// lazy path exactly as before, which is what keeps "refresh failed" from meaning "no key". + /// + private volatile IReadOnlyDictionary _refreshed = + new Dictionary(StringComparer.OrdinalIgnoreCase); + /// /// The known-cluster gate's membership set, or when the gate is /// inactive (never published, or published empty — see ). @@ -210,6 +241,13 @@ public sealed class MeshPskProvider : IMeshPskProvider return true; } + // Pre-resolved by the asynchronous refresh: no store read on this (gRPC request) thread at all. + if (_refreshed.TryGetValue(clusterId, out var refreshed)) + { + psk = refreshed; + return true; + } + var entry = _cache.GetOrAdd( clusterId, static (id, self) => new Lazy( @@ -288,7 +326,124 @@ public sealed class MeshPskProvider : IMeshPskProvider } /// - public void InvalidateAll() => _cache.Clear(); + public void InvalidateAll() + { + _cache.Clear(); + _refreshed = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + /// + /// Ceiling on concurrent store reads during one . A fleet of N clusters used to + /// fan out N simultaneous reads every contact refresh; the store (and anything behind it) sees at most + /// this many at once now. + /// + public const int MaxConcurrentRefreshReads = 8; + + /// + public async Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(clusterIds); + + var previous = _refreshed; + var ids = clusterIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + var results = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + + // Bounded fan-out; each read bounded like the synchronous path. + await Parallel.ForEachAsync( + ids, + new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRefreshReads, CancellationToken = ct }, + async (id, token) => results[id] = await ResolveFromStoreAsync(id, token).ConfigureAwait(false)) + .ConfigureAwait(false); + + // Three outcomes per cluster (fix round 1 / I8): + // - Found → the new key; + // - NotFound → the store answered that the secret does not exist: a REVOCATION — the cluster gets no + // key (and its lazily cached one is evicted below), so a deleted secret stops working + // within one refresh instead of living on for the life of the process; + // - Failed → the store could not be read (threw / timed out / unnameable): keep the PREVIOUS key — + // a transient fault must not open a window in which the cluster has no key at all. + var next = new Dictionary(StringComparer.OrdinalIgnoreCase); + var answered = new List(); + foreach (var id in ids) + { + var read = results[id]; + switch (read.Outcome) + { + case StoreReadOutcome.Found: + next[id] = read.Value!; + answered.Add(id); + break; + case StoreReadOutcome.NotFound: + answered.Add(id); + break; + default: + if (previous.TryGetValue(id, out var kept)) next[id] = kept; + break; + } + } + + // The swap. Then drop the lazy entries for every cluster the store ANSWERED for — found (serve the new + // snapshot, not a stale lazy resolution) or not found (revoked: a lazily cached key must die too). + // Entries for clusters whose read failed are left alone. + _refreshed = next; + foreach (var id in answered) _cache.TryRemove(id, out _); + } + + private enum StoreReadOutcome { Found, NotFound, Failed } + + private readonly record struct StoreRead(StoreReadOutcome Outcome, string? Value); + + /// + /// The asynchronous twin of : same name derivation, same bound — awaited + /// instead of blocked on, because this one runs on a refresh rather than inside a gRPC interceptor — + /// but it keeps "the secret does not exist" (a blank answer) apart from "the store could not be read". + /// + /// The application Cluster id. + /// The caller's token, linked under the store-read bound. + /// The outcome and, when found, the key. + private async Task ResolveFromStoreAsync(string clusterId, CancellationToken ct) + { + SecretName name; + try + { + name = new SecretName(SecretNamePrefix + clusterId); + } + catch (ArgumentException ex) + { + _logger.LogWarning( + ex, + "Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be " + + "refreshed from the store. Supply it through MeshTransport:ClusterPsks instead, or " + + "rename the cluster to [a-z0-9._/-].", + clusterId); + return new StoreRead(StoreReadOutcome.Failed, null); + } + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + cts.CancelAfter(StoreReadTimeout); + try + { + var value = await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(value)) return new StoreRead(StoreReadOutcome.Found, value); + + _logger.LogWarning( + "Mesh PSK secret {SecretName} for cluster {ClusterId} is absent (or blank) in the store; the key is " + + "REVOKED — commands to that cluster and acks from it are refused until it is restored.", + name.Value, + clusterId); + return new StoreRead(StoreReadOutcome.NotFound, null); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Failed to refresh the mesh PSK for cluster {ClusterId} from secret {SecretName}; the " + + "previously resolved key (if any) stays in service until the next refresh.", + clusterId, + name.Value); + return new StoreRead(StoreReadOutcome.Failed, null); + } + } /// /// Reads OT-MESH-PSK-{clusterId} from the Secrets store, converting every failure mode diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs index a10d228a..31a19b13 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs @@ -34,6 +34,15 @@ public interface IMeshCommandDialer /// rather than dialled unauthenticated. /// Task DispatchAsync(string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct); + + /// + /// Releases any per-endpoint resources for endpoints NOT in + /// (R03:1.3). Central calls it on every contact refresh with the current dispatch set, so a + /// renamed host or moved port does not leave the old channel and its idle HTTP/2 connection + /// alive for the life of the process. Default no-op for dialers that cache nothing. + /// + /// The endpoints still in the dispatch set. + void RetainOnly(IReadOnlyCollection nodeEndpoints) { } } /// @@ -181,6 +190,30 @@ public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable await call.ResponseAsync.ConfigureAwait(false); } + /// + public void RetainOnly(IReadOnlyCollection nodeEndpoints) + { + ArgumentNullException.ThrowIfNull(nodeEndpoints); + if (_disposed) return; + + var keep = nodeEndpoints as IReadOnlySet ?? nodeEndpoints.ToHashSet(StringComparer.Ordinal); + foreach (var endpoint in _channels.Keys) + { + if (keep.Contains(endpoint)) continue; + // A dial that already took this channel out of the cache before the removal fails once, as + // a named per-node RpcException the actor logs; the next dial to that endpoint (if it ever + // returns to the set) builds a fresh channel. Bounded to the dials in flight at the instant + // of a contact refresh, the same accepted race Dispose documents. + if (_channels.TryRemove(endpoint, out var channel)) + { + channel.Dispose(); + _log.LogInformation( + "Mesh dialer released the cached channel for {Endpoint}: it is no longer in the dispatch set", + endpoint); + } + } + } + /// Marks the dialer disposed and disposes every cached gRPC channel. public void Dispose() { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index b621cc64..d408343c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -161,9 +161,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(DeploymentAcksTopic, Self)); using var db = _dbFactory.CreateDbContext(); + // Projected: the row carries the whole ArtifactBlob (varbinary(max)), which this recovery never + // reads — and this query runs on every singleton start and hand-over. var inflight = db.Deployments .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, d.RevisionHash, d.CreatedAtUtc }) .FirstOrDefault(); if (inflight is null) return; @@ -173,6 +176,41 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers .AsNoTracking() .ToList(); + // Resume the deadline timer using the remaining time. The deadline runs from when the + // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough + // proxy — we don't track a separate "dispatched at" column). + var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc; + var remaining = _applyDeadline - elapsed; + + if (nodeStates.Count == 0) + { + // The expected set and the AwaitingApplyAcks flip are written in ONE SaveChanges by + // HandleDispatch, so a row with no NodeDeploymentState rows is a dispatch that never completed: + // HandleDispatch threw before its SaveChanges (this actor restarted and the message was + // dropped), the process died between AdminOps' row write and the coordinator's Tell, or the + // Tell was lost across a singleton hand-over. Recovering it with an EMPTY expected set used to + // let the deadline take the "everyone acked" race branch (0 == 0) and leave the row in flight + // forever, refusing every later StartDeployment. Inside the deadline the dispatch is simply + // re-driven through the normal handler (bounded: CreatedAtUtc never moves, so a dispatch that + // keeps throwing ends up on the other branch once the deadline passes); past it the row is + // closed as TimedOut with a reason that names the condition. + if (remaining > TimeSpan.Zero) + { + _log.Warning( + "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " + + "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " + + "re-dispatching with {Remaining} of the apply deadline left", + _current, remaining); + Self.Tell(new DispatchDeployment( + _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId())); + } + else + { + Self.Tell(new DeadlineElapsed(_current.Value)); + } + return; + } + _expectedAcks = nodeStates.Select(s => NodeId.Parse(s.NodeId)).ToHashSet(NodeIdComparer); foreach (var s in nodeStates.Where(s => s.Status != NodeDeploymentStatus.Applying)) { @@ -182,11 +220,6 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId)); } - // Resume the deadline timer using the remaining time. The deadline runs from when the - // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough - // proxy — we don't track a separate "dispatched at" column). - var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc; - var remaining = _applyDeadline - elapsed; if (remaining <= TimeSpan.Zero) { Self.Tell(new DeadlineElapsed(_current.Value)); @@ -201,6 +234,33 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers private void HandleDispatch(DispatchDeployment msg) { + // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already + // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message + // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it. + // + // The RowVersion read HERE is the one the AwaitingApplyAcks flip below is conditioned on (re-review 1): + // reading a fresh one at flip time left a window in which AdminOps' lost-dispatch closure could commit + // TimedOut between the guard and the flip, and the flip would silently reopen the row. With the guard's + // RowVersion, any change since the guard read makes the flip a concurrency conflict instead. + byte[]? guardRowVersion; + using (var check = _dbFactory.CreateDbContext()) + { + var guard = check.Deployments + .AsNoTracking() + .Where(d => d.DeploymentId == msg.DeploymentId.Value) + .Select(d => new { d.Status, d.RowVersion }) + .FirstOrDefault(); + if (guard?.Status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) + { + _log.Warning( + "Ignoring DispatchDeployment {Id}: the deployment is already {Status} (a late or duplicated dispatch)", + msg.DeploymentId, guard.Status); + if (_current == msg.DeploymentId) ResetForNext(); + return; + } + guardRowVersion = guard?.RowVersion; + } + _current = msg.DeploymentId; _acks.Clear(); _degraded.Clear(); @@ -208,22 +268,51 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // Seed NodeDeploymentState rows so a failover knows which nodes were expected to ack. using (var db = _dbFactory.CreateDbContext()) { - // Discovery reuses this context rather than opening a second one — it reads the same - // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single - // context keeps the expected set and the seeded rows consistent by construction. - _expectedAcks = DiscoverDriverNodes(db); - - foreach (var node in _expectedAcks) + // Idempotent on a re-dispatch (PreStart re-drives a lost dispatch through this handler): rows + // already seeded for this deployment ARE the expected set — re-discovering would both duplicate + // the primary key and let a node toggled since the first attempt change who is expected. + var seeded = db.NodeDeploymentStates + .AsNoTracking() + .Where(x => x.DeploymentId == msg.DeploymentId.Value) + .Select(x => x.NodeId) + .ToList(); + if (seeded.Count > 0) { - db.NodeDeploymentStates.Add(new NodeDeploymentState - { - NodeId = node.Value, - DeploymentId = msg.DeploymentId.Value, - Status = NodeDeploymentStatus.Applying, - }); + _expectedAcks = seeded.Select(NodeId.Parse).ToHashSet(NodeIdComparer); + } + else + { + // Discovery reuses this context rather than opening a second one — it reads the same + // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single + // context keeps the expected set and the seeded rows consistent by construction. + _expectedAcks = DiscoverDriverNodes(db); + + foreach (var node in _expectedAcks) + { + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = node.Value, + DeploymentId = msg.DeploymentId.Value, + Status = NodeDeploymentStatus.Applying, + }); + } + } + UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks, rowVersion: guardRowVersion); + try + { + db.SaveChanges(); + } + catch (DbUpdateConcurrencyException) + { + // The row moved after the guard read (on SQL Server: AdminOps closed it as a lost dispatch). The + // expected-set rows and the flip were one SaveChanges, so nothing was written; nothing is + // published and the coordinator is free for the next deployment. + _log.Warning( + "DispatchDeployment {Id} abandoned: the deployment changed state while it was being dispatched " + + "(closed as a lost dispatch?) — not reopening it", msg.DeploymentId); + ResetForNext(); + return; } - UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks); - db.SaveChanges(); } // Per-cluster mesh Phase 2: hand off to the transport router rather than publishing @@ -323,11 +412,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // same as clean, though: the deployment's reason names the degraded nodes (per-node detail is on // each NodeDeploymentState row) and the seal logs at Warning. var nodes = _degraded.Select(n => n.Value).Order(StringComparer.OrdinalIgnoreCase).ToList(); - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); - if (d is not null) - d.FailureReason = Truncate( - $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " + - "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it."); + SetFailureReason(db, Truncate( + $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " + + "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it.")); db.SaveChanges(); _log.Warning( "Deployment {Id} sealed DEGRADED (acks={Count}) — the address-space apply was degraded on: {Nodes}", @@ -360,7 +447,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers msg.DeploymentId, _current); return; } - if (_acks.Count == _expectedAcks.Count) + // Only a NON-EMPTY expected set can be "fully acked": with nothing expected, 0 == 0 is not the + // everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped. + if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count) { // Race: every node acked just as the deadline fired. Already sealed/failed elsewhere. return; @@ -368,6 +457,19 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers using var db = _dbFactory.CreateDbContext(); UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut); + if (_expectedAcks.Count == 0) + { + SetFailureReason(db, Truncate( + "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + + "hand-over mid-dispatch) and the apply deadline has passed. Redeploy.")); + db.SaveChanges(); + _log.Warning( + "Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " + + "expected and the {Deadline} apply deadline has passed. Closed as TimedOut — redeploy", + _current.Value, _applyDeadline); + ResetForNext(); + return; + } db.SaveChanges(); // Name the silent nodes, not just the counts. Since the expected set became DB-derived this // is the failure an operator meets after deploying while a configured node is switched off, @@ -394,13 +496,63 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers public sealed record DeadlineElapsed(DeploymentId DeploymentId); + /// + /// Stages a status change WITHOUT materialising the row: the row carries the whole ArtifactBlob + /// (varbinary(max), multi-MB on a real fleet), which no status transition reads (fix round 1). A stub + /// carrying only the key and the row's current RowVersion is attached and the changed columns marked + /// modified, so the UPDATE writes those columns and keeps the optimistic-concurrency check. The staged + /// stub stays tracked so can add to the same UPDATE. + /// private static void UpdateDeploymentStatus( - OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false) + OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false, byte[]? rowVersion = null) { - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == id.Value); - if (d is null) return; - d.Status = status; - if (sealNow) d.SealedAtUtc = DateTime.UtcNow; + var entry = TrackStub(db, id, rowVersion); + if (entry is null) return; + entry.Entity.Status = status; + entry.Property(x => x.Status).IsModified = true; + if (sealNow) + { + entry.Entity.SealedAtUtc = DateTime.UtcNow; + entry.Property(x => x.SealedAtUtc).IsModified = true; + } + } + + /// Stages the current deployment's failure reason onto the stub staged. + private void SetFailureReason(OtOpcUaConfigDbContext db, string reason) + { + if (_current is null) return; + var entry = TrackStub(db, _current.Value); + if (entry is null) return; + entry.Entity.FailureReason = reason; + entry.Property(x => x.FailureReason).IsModified = true; + } + + /// The tracked blob-free stub for a deployment, attaching one on first use; null when the row is gone. + /// The context. + /// The deployment. + /// + /// The RowVersion the update must be conditioned on, when the caller already read one (the dispatch + /// guard); reads the current one. + /// + private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry? TrackStub( + OtOpcUaConfigDbContext db, DeploymentId id, byte[]? rowVersion = null) + { + var tracked = db.ChangeTracker.Entries().FirstOrDefault(e => e.Entity.DeploymentId == id.Value); + if (tracked is not null) return tracked; + if (rowVersion is null) + { + var stamp = db.Deployments + .AsNoTracking() + .Where(x => x.DeploymentId == id.Value) + .Select(x => new { x.RowVersion }) + .FirstOrDefault(); + if (stamp is null) return null; + rowVersion = stamp.RowVersion; + } + return db.Deployments.Attach(new Deployment + { + DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = rowVersion, + }); } /// diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs new file mode 100644 index 00000000..67020803 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs @@ -0,0 +1,190 @@ +using Akka.Actor; +using Akka.Event; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; + +/// +/// Admin-role cluster singleton that periodically empties the ArtifactBlob of old, terminal +/// deployments (R03:1.2). See for the policy and +/// for the rule, which is pure over a context so it is unit-testable. +/// +/// +/// +/// Empties, never deletes, and never nulls. The column is NOT NULL varbinary(max), and +/// every reader (DriverHostActor.LoadArtifactBlob, OpcUaPublishActor, +/// DeploymentArtifactService) already treats a zero-length blob as "no artifact" (#485: empty +/// bytes are not an empty configuration — the apply fails and the node keeps last-known-good). An +/// empty blob therefore needs no schema change and no reader change, where a NULL would need both. +/// +/// +/// Never touches an in-flight row (Dispatching / AwaitingApplyAcks): a node may still be +/// fetching it. The newest terminal rows keep +/// their blob whatever their age, so the "latest sealed" every boot path reads is always intact. +/// +/// +public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers +{ + private readonly IDbContextFactory _dbFactory; + private readonly DeploymentRetentionOptions _options; + private readonly ILoggingAdapter _log = Context.GetLogger(); + private const string SweepTimerKey = "retention-sweep"; + + /// Gets or sets the timer scheduler. + public ITimerScheduler Timers { get; set; } = null!; + + /// Creates the actor props. + /// The config DB factory. + /// The retention policy. + /// The props. + public static Props Props(IDbContextFactory dbFactory, DeploymentRetentionOptions options) => + Akka.Actor.Props.Create(() => new DeploymentRetentionActor(dbFactory, options)); + + /// Initializes a new instance of the class. + /// The config DB factory. + /// The retention policy. + public DeploymentRetentionActor(IDbContextFactory dbFactory, DeploymentRetentionOptions options) + { + ArgumentNullException.ThrowIfNull(dbFactory); + ArgumentNullException.ThrowIfNull(options); + _dbFactory = dbFactory; + _options = options; + + Receive(_ => RunSweep()); + Receive(msg => + { + if (msg.Pruned > 0) + _log.Info("Deployment retention emptied the artifact blob on {Count} old deployment(s)", msg.Pruned); + else + _log.Debug("Deployment retention sweep found nothing to prune"); + if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg); + }); + Receive(msg => + { + _log.Warning(msg.Error, "Deployment retention sweep failed; the next sweep retries"); + if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg); + }); + } + + /// + protected override void PreStart() + { + if (!_options.Enabled) + { + _log.Info("Deployment retention is disabled (Deployments:Retention:Enabled=false); artifact blobs are kept forever"); + return; + } + var interval = TimeSpan.FromMinutes(Math.Max(1, _options.IntervalMinutes)); + var initial = TimeSpan.FromSeconds(Math.Max(0, _options.InitialDelaySeconds)); + Timers.StartPeriodicTimer(SweepTimerKey, Sweep.Instance, initial, interval); + } + + private void RunSweep() + { + var replyTo = Sender.IsNobody() ? null : Sender; + var self = Self; + var now = DateTime.UtcNow; + Task.Run(async () => + { + await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false); + return await SweepAsync(db, _options, now).ConfigureAwait(false); + }) + .PipeTo(self, + success: pruned => new SweepCompleted(pruned, replyTo), + failure: ex => new SweepFailed(ex, replyTo)); + } + + /// + /// The retention rule. Empties the blob of every terminal deployment that is (a) not one of the + /// newest SEALED rows (at least one — the newest + /// Sealed blob is never pruned), (b) not referenced by a + /// NodeDeploymentState row started or applied within + /// , and (c) still carrying bytes — at most + /// per call. + /// + /// The config DB. + /// The policy. + /// The sweep's clock. + /// Cancellation. + /// How many rows were emptied. + public static async Task SweepAsync( + OtOpcUaConfigDbContext db, DeploymentRetentionOptions options, DateTime utcNow, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(options); + + var keep = Math.Max(1, options.KeepSealed); + var referenceFloor = utcNow.AddDays(-Math.Max(0, options.ReferencedDays)); + var batch = Math.Max(1, options.MaxRowsPerSweep); + + // Projected: ids only. The candidate scan must not materialise blobs for rows it will keep. + var terminal = await db.Deployments + .AsNoTracking() + .Where(d => d.Status == DeploymentStatus.Sealed + || d.Status == DeploymentStatus.PartiallyFailed + || d.Status == DeploymentStatus.TimedOut) + .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, d.Status, HasBlob = d.ArtifactBlob.Length > 0 }) + .ToListAsync(ct) + .ConfigureAwait(false); + + // The keep window counts SEALED rows only (fix round 1 / I3). Counting every terminal row let a burst + // of failed deploys (TimedOut / PartiallyFailed) fill the window by itself and push the fleet's last + // good artifact — the newest Sealed row, which boot, drift detection and the publish fallback all + // read — out of it. `keep` is clamped to ≥ 1, so the newest Sealed row is never a candidate. + var keptSealed = terminal + .Where(d => d.Status == DeploymentStatus.Sealed) + .Take(keep) + .Select(d => d.DeploymentId) + .ToHashSet(); + var aged = terminal + .Where(d => d.HasBlob && !keptSealed.Contains(d.DeploymentId)) + .Select(d => d.DeploymentId) + .ToList(); + if (aged.Count == 0) return 0; + + var referenced = (await db.NodeDeploymentStates + .AsNoTracking() + .Where(s => aged.Contains(s.DeploymentId) + && (s.StartedAtUtc >= referenceFloor + || (s.AppliedAtUtc != null && s.AppliedAtUtc >= referenceFloor))) + .Select(s => s.DeploymentId) + .Distinct() + .ToListAsync(ct) + .ConfigureAwait(false)) + .ToHashSet(); + + var targets = aged.Where(id => !referenced.Contains(id)).Take(batch).ToList(); + if (targets.Count == 0) return 0; + + // Tracked load of ONLY the rows being emptied: the update rides the row's real RowVersion on + // both providers, and the blob read is the one-time cost of retiring that row. + var rows = await db.Deployments + .Where(d => targets.Contains(d.DeploymentId)) + .ToListAsync(ct) + .ConfigureAwait(false); + foreach (var row in rows) row.ArtifactBlob = []; + await db.SaveChangesAsync(ct).ConfigureAwait(false); + return rows.Count; + } + + /// Asks for a sweep now; the sender (if any) receives the / . + public sealed class Sweep + { + /// The singleton instance. + public static readonly Sweep Instance = new(); + private Sweep() { } + } + + /// A sweep finished. + /// How many rows were emptied. + /// Who asked, when the sweep was requested rather than timer-driven. + public sealed record SweepCompleted(int Pruned, IActorRef? ReplyTo); + + /// A sweep threw. + /// The failure. + /// Who asked, when the sweep was requested rather than timer-driven. + public sealed record SweepFailed(Exception Error, IActorRef? ReplyTo); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs new file mode 100644 index 00000000..6bdc571f --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs @@ -0,0 +1,55 @@ +using Microsoft.Extensions.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; + +/// +/// Deployments:Retention — how long a sealed deployment keeps its ArtifactBlob (R03:1.2). +/// Every deploy stored the whole gzip'd fleet snapshot forever; nothing pruned it, so a year of routine +/// edits was gigabytes of blobs. The retention sweep EMPTIES the blob on old rows and never deletes a +/// row: the audit record (who deployed what revision, when, and how every node acked) survives. +/// +public sealed class DeploymentRetentionOptions +{ + /// The configuration section name. + public const string SectionName = "Deployments:Retention"; + + /// Master switch; keeps every blob forever (the previous behaviour). + public bool Enabled { get; set; } = true; + + /// + /// How many of the most recent SEALED deployments keep their blob regardless of age (clamped to ≥ 1). + /// Failed deploys (PartiallyFailed / TimedOut) do not count toward it, so a burst of them can never push + /// the last good artifact out. A node booting from central SQL reads the LATEST sealed row's blob, so + /// this must stay well above 1; 50 also keeps a comfortable re-deploy history. + /// + public int KeepSealed { get; set; } = 50; + + /// + /// A deployment any NodeDeploymentState row started or applied within this many days keeps its + /// blob even when older than the window — a node that just applied it may + /// still replay it on restart. + /// + public int ReferencedDays { get; set; } = 30; + + /// How often the sweep runs. + public int IntervalMinutes { get; set; } = 60; + + /// Delay before the first sweep after the singleton starts, so a boot is not a blob scan. + public int InitialDelaySeconds { get; set; } = 60; + + /// + /// Rows emptied per sweep. The sweep loads each candidate row (blob included) to update it under its + /// RowVersion, so this bounds one sweep's memory and transaction size; a backlog drains over sweeps. + /// + public int MaxRowsPerSweep { get; set; } = 200; + + /// Binds the section from configuration, with defaults for anything absent. + /// The app configuration; null yields the defaults. + /// The bound options. + public static DeploymentRetentionOptions Bind(IConfiguration? configuration) + { + var options = new DeploymentRetentionOptions(); + configuration?.GetSection(SectionName).Bind(options); + return options; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index cb8d605a..9a9f0742 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; @@ -15,6 +16,7 @@ using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.ControlPlane; @@ -30,6 +32,7 @@ public static class ServiceCollectionExtensions public const string AuditWriterSingletonName = "audit-writer"; public const string RedundancyStateSingletonName = "redundancy-state"; public const string ClusterNodeAddressReconcilerSingletonName = "cluster-node-address-reconciler"; + public const string DeploymentRetentionSingletonName = "deployment-retention"; /// /// Registers the admin-role cluster singletons on the AkkaConfigurationBuilder — three with @@ -109,7 +112,7 @@ public static class ServiceCollectionExtensions // coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks // inside a props factory and races the spawn it is waiting for. var meshRouter = registry.Get(); - return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: meshRouter); + return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: DeployTimings.ApplyDeadline, meshRouter: meshRouter); }, singletonOptions); @@ -126,7 +129,10 @@ public static class ServiceCollectionExtensions resolver.GetService()?["Deployment:TagConfigValidationMode"], ignoreCase: true, out var m) ? m : TagConfigValidationMode.Warn; var meshRouter = registry.Get(); - return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter); + // The SAME deadline source as the coordinator above: AdminOps closes a lost dispatch only once it + // is past the coordinator's deadline, so the two must never drift. + return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter, + applyDeadline: DeployTimings.ApplyDeadline); }, singletonOptions); @@ -181,6 +187,21 @@ public static class ServiceCollectionExtensions singletonOptions, createProxyToo: false); + // Deployment artifact retention (R03:1.2): empties ArtifactBlob on old terminal deployments so the + // Deployment table stops growing by one fleet snapshot per edit. Admin singleton for the same + // reason as the reconciler (one sweeper per fleet, and only central holds the ConfigDb); + // proxy-less because nothing addresses it in production — a test sends Sweep directly. + builder.WithSingleton( + DeploymentRetentionSingletonName, + (system, registry, resolver) => + { + var dbFactory = resolver.GetService>(); + var options = DeploymentRetentionOptions.Bind(resolver.GetService()); + return DeploymentRetentionActor.Props(dbFactory, options); + }, + singletonOptions, + createProxyToo: false); + return builder; } @@ -256,4 +277,5 @@ public sealed class AdminOperationsActorKey { } public sealed class AuditWriterActorKey { } public sealed class RedundancyStateActorKey { } public sealed class ClusterNodeAddressReconcilerKey { } +public sealed class DeploymentRetentionKey { } public sealed class CentralCommunicationActorKey { } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs new file mode 100644 index 00000000..eced66a8 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs @@ -0,0 +1,41 @@ +using ZB.MOM.WW.Configuration; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; + +namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; + +/// +/// Fail-fast startup validator for (R03:2.3). The pepper used to be +/// checked only when the first key was verified — ApiKeySecret.Verify throws on an empty one — +/// so a site node deployed without Security__ApiKeys__Pepper booted green, passed health, +/// applied deployments and denied every machine client with the same opaque "Invalid API key" until +/// someone read the log; a central node with a wrong pepper minted keys that could verify nowhere. +/// Every other security section already fails host start on a bad value; this one now does too. +/// +/// +/// Registered for hasDriver || hasAdmin: a driver node needs the pepper to VERIFY keys, an admin +/// node to MINT them. The minimum length is a floor against a placeholder ("x") passing as +/// configured, not a strength claim — the pepper is a server-side secret, not a password. +/// +public sealed class ApiKeyOptionsValidator : OptionsValidatorBase +{ + /// The shortest pepper accepted (characters, after trimming). + public const int MinPepperLength = 16; + + /// + protected override void Validate(ValidationBuilder builder, ApiKeyOptions options) + { + var pepper = options.Pepper?.Trim() ?? string.Empty; + builder.RequireThat(pepper.Length > 0, + $"{ApiKeyOptions.SectionName}:Pepper is required on any node with the driver or admin role — " + + "supply it via Security__ApiKeys__Pepper or a ${secret:} reference. An empty pepper would reject " + + "every API key (driver) or mint keys that can never verify (admin)."); + builder.RequireThat(pepper.Length == 0 || pepper.Length >= MinPepperLength, + $"{ApiKeyOptions.SectionName}:Pepper must be at least {MinPepperLength} characters."); + builder.RequireThat(!string.IsNullOrWhiteSpace(options.Username), + $"{ApiKeyOptions.SectionName}:Username must not be blank."); + builder.RequireThat(options.StoreTimeoutMs >= 0, + $"{ApiKeyOptions.SectionName}:StoreTimeoutMs must be >= 0 (0 disables the deadline)."); + builder.RequireThat(options.StoreCacheSeconds >= 0, + $"{ApiKeyOptions.SectionName}:StoreCacheSeconds must be >= 0 (0 disables the cache)."); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs index e7c301e4..eace8505 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs @@ -80,7 +80,10 @@ public static class DataPlaneAuthRegistration sp.GetRequiredService>().Value)), sp.GetRequiredService>(), sp.GetRequiredService().CreateLogger(), - sp.GetRequiredService())); + sp.GetRequiredService(), + // R03:3.3 — the known-id set + miss budget in front of the SQL lookup; same section as + // ApiKeyOptions, Host-owned knobs. + ApiKeyStoreGuardOptions.Bind(sp.GetService()))); services.AddSingleton(); // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs index d77631ea..8a9347d9 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs @@ -31,8 +31,17 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines; /// into the fresh cache for one generation; P7's expression-set gate means no-op redeploys no /// longer open this window at all. /// +/// +/// Run quarantine. Every compiled run goes through one keyed by +/// virtual-tag id with the expression as the source identity: a run is refused while the tag's previous +/// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive +/// timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach. Both surface as +/// so VirtualTagActor publishes Bad exactly as it does for a timeout. A quarantine lifts when the +/// expression changes, on / (the deploy +/// boundary), never by itself. +/// /// -public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IDisposable +public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IScriptQuarantineOwner, IDisposable { // Purpose-built source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and // Clear()/Dispose() that dispose each evaluator's collectible AssemblyLoadContext. The apply-boundary @@ -42,22 +51,29 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac private readonly ILogger _logger; private readonly SerilogLogger _scriptRoot; private readonly TimeSpan _runTimeout; + private readonly ScriptRunGuard _guard; private bool _disposed; /// Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout. /// Logger for recording compilation and execution errors (host diagnostics). /// Root script logger; user ctx.Logger.* output flows through this to the Script-log page. /// Maximum execution time for each script; defaults to 2 seconds if not specified. + /// Run-guard budget + quarantine threshold (the Scripting section); null ⇒ defaults. public RoslynVirtualTagEvaluator( ILogger logger, ScriptRootLogger scriptRoot, - TimeSpan? runTimeout = null) + TimeSpan? runTimeout = null, + ScriptingOptions? scripting = null) { _logger = logger; _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger; _runTimeout = runTimeout ?? TimeSpan.FromSeconds(2); + _guard = new ScriptRunGuard(scripting, _scriptRoot); } + /// The run guard every compiled evaluation goes through (diagnostics / tests). + public ScriptRunGuard RunGuard => _guard; + // Value-only overload. Every dependency reaches the script stamped Good with an // evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot // overload instead; this one remains for callers that genuinely hold only @@ -150,13 +166,13 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac try { - // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't - // interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous - // CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when - // the wall-clock budget fires regardless of whether the inner task completes (the orphaned - // thread is the documented, accepted trade-off — TimedScriptEvaluator remarks). - var timed = new TimedScriptEvaluator(evaluator, _runTimeout); - var raw = timed.RunAsync(context).GetAwaiter().GetResult(); + // Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by virtual-tag id): a raw + // CancellationToken can't interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll + // it), so the wall-clock budget is what returns control — and the guard is what stops the + // abandoned run from being joined by another on the next trigger (one orphan per tag, then + // quarantine), which is the leak the old per-evaluation TimedScriptEvaluator allowed. + var raw = _guard.RunAsync(virtualTagId, expression, evaluator, context, _runTimeout) + .GetAwaiter().GetResult(); return VirtualTagEvalResult.Ok(raw); } catch (ScriptTimeoutException) @@ -164,6 +180,14 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac return VirtualTagEvalResult.Failure( $"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s"); } + catch (ScriptRunRefusedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } + catch (ScriptQuarantinedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } catch (ObjectDisposedException) when (!_disposed && attempt == 0) { // S12: the apply-boundary ClearCompiledScripts disposed the evaluator between our @@ -185,7 +209,19 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac } /// - public void ClearCompiledScripts() => _cache.Clear(); + public void ClearCompiledScripts() + { + _cache.Clear(); + // The compile-cache clear IS the deploy boundary for this evaluator: a redeploy lifts every quarantine + // (an edited expression already lifts its own through the guard's source identity). + _guard.ResetAll(); + } + + /// + public void ResetQuarantines() => _guard.ResetAll(); + + /// + public IReadOnlyCollection QuarantinedScripts => _guard.QuarantinedKeys; /// Disposes the evaluator and cleans up all cached script evaluators. public void Dispose() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs index f7a740bd..824223dd 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs @@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Host.Configuration; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; namespace ZB.MOM.WW.OtOpcUa.Host.Health; @@ -28,11 +29,40 @@ public static class HealthEndpoints /// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to /// true so admin/fused callers (including the test harnesses) keep the probe unchanged. /// + /// + /// Whether this node carries the driver role and therefore serves an OPC UA endpoint. When + /// true the joins the ready tier, and the active tier on a + /// driver-ONLY node (R03:2.1; on a fused node the active tier routes the AdminUI — see the body), reading + /// the the host registers alongside OtOpcUaServerHostedService. + /// Defaults to false so admin-only callers and the existing harnesses register nothing new. + /// /// The same service collection, for chaining. - public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true) + public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false) { var checks = services.AddHealthChecks(); + // A failed SDK start takes the node out of the READY tier on every driver node: it cannot serve. + // The ACTIVE tier depends on the roles (coordinator ruling, fix round 1 / I6): + // - driver-only: Active too — a site node's whole purpose is its OPC UA endpoint, so nothing may + // route to it; + // - fused admin+driver: NOT Active, until the ServiceLevel calculator consumes IOpcUaEndpointState + // (follow-up F-C1). Traefik routes the AdminUI on /health/active, so putting the probe there would + // take the operator's only console out of rotation precisely when they need it to diagnose the + // failed endpoint, while the OPC UA failover (ServiceLevel) still would not move. Once F-C1 lands + // the peer takes Primary and the active tier follows it on its own. + // Factory-registered so the state singleton is resolved per probe. + if (hasDriver) + { + var tags = hasAdmin + ? new[] { ZbHealthTags.Ready } + : new[] { ZbHealthTags.Ready, ZbHealthTags.Active }; + checks.Add(new HealthCheckRegistration( + OpcUaEndpointHealthCheck.Name, + sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService()), + failureStatus: null, + tags: tags)); + } + // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config // from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so // the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs new file mode 100644 index 00000000..e619d7b3 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs @@ -0,0 +1,47 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Health; + +/// +/// Ready-tier probe (plus Active on a driver-only node) for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is +/// listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy +/// one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets +/// connection-refused. +/// +/// +/// Registered on driver-role nodes only (); an +/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. On a fused +/// admin+driver node it is Ready-only until ServiceLevel consumes the fact (F-C1), because the Active tier +/// is what Traefik routes the AdminUI on. +/// +public sealed class OpcUaEndpointHealthCheck : IHealthCheck +{ + /// The registration name, as it appears in the health JSON. + public const string Name = "opcua-endpoint"; + + private readonly IOpcUaEndpointState _state; + + /// Initializes a new instance of the class. + /// The endpoint lifecycle fact the hosted service maintains. + public OpcUaEndpointHealthCheck(IOpcUaEndpointState state) + { + ArgumentNullException.ThrowIfNull(state); + _state = state; + } + + /// + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + var data = new Dictionary { ["status"] = _state.Status.ToString() }; + var result = _state.Status switch + { + OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data), + OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data), + OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data), + _ => HealthCheckResult.Unhealthy( + "OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data), + }; + return Task.FromResult(result); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs new file mode 100644 index 00000000..3103b982 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs @@ -0,0 +1,56 @@ +using Microsoft.Extensions.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// +/// The pre-authentication guards puts in front of the ConfigDb key +/// lookup (R03:3.3). Bound from the same Security:ApiKeys section as ApiKeyOptions but +/// owned by the Host, where the store lives. +/// +/// +/// +/// Why. The negative cache was keyed by key id, so a flood of DISTINCT well-formed +/// otua_<random>_x credentials missed it every time and cost one indexed SQL query per +/// attempt — under the SDK's SessionManager-wide lock — with no credential needed. Two guards close +/// that: a known-id set (every ApiKeyId the ConfigDb holds, refreshed every +/// StoreCacheSeconds) rejects an unknown id in memory, and a miss budget (token +/// bucket) bounds how many lookups that reach the store and MISS are allowed per second when the +/// set is unavailable. +/// +/// +/// The budget is process-wide, not per remote endpoint: the OPC UA SDK's impersonation callback +/// (the IOpcUaUserAuthenticator seam) does not carry the client's address. Per-endpoint +/// accounting needs that seam widened in the OpcUaServer project — a follow-up, not this lane. +/// +/// +public sealed class ApiKeyStoreGuardOptions +{ + /// The configuration section the knobs live in (shared with ApiKeyOptions). + public const string SectionName = "Security:ApiKeys"; + + /// + /// Whether the known-id set is consulted. restores a store read per unknown + /// id. Only meaningful when the inner store can list ids (the ConfigDb store can; the artifact + /// store is never wrapped). + /// + public bool KnownIdSet { get; set; } = true; + + /// + /// How many store MISSES (lookups that reach the inner store and find no key) may burst before + /// further lookups for unknown ids are refused without a store read. 0 disables the budget. + /// + public int MissBurst { get; set; } = 100; + + /// How many miss tokens are restored per second, up to . + public double MissRefillPerSecond { get; set; } = 20; + + /// Binds the section, with defaults for anything absent. + /// The app configuration; null yields the defaults. + /// The bound options. + public static ApiKeyStoreGuardOptions Bind(IConfiguration? configuration) + { + var options = new ApiKeyStoreGuardOptions(); + configuration?.GetSection(SectionName).Bind(options); + return options; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index af0fca3f..6ebb7626 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -56,6 +56,33 @@ public sealed class BoundedApiKeyStore : IApiKeyStore private readonly ConcurrentDictionary> _keys = new(StringComparer.Ordinal); private readonly ConcurrentDictionary>> _grants = new(StringComparer.Ordinal); + private readonly ApiKeyStoreGuardOptions _guard; + + // R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds + // (single-flight; a refresh that fails or times out keeps the previous set). Null until the first + // successful load, or when the inner store cannot list ids — then the per-id path runs as before. + // The set and the instant it was loaded are published TOGETHER as one immutable snapshot behind a volatile + // reference (fix round 1): a reader on another thread can never pair a new set with an old stamp, or read + // a torn DateTimeOffset (a 16-byte struct, not atomically readable), as the separate fields allowed. + private volatile KnownIdSnapshot? _knownIds; + private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1); + + // Set when the inner store reports it cannot list ids. Per-instance on purpose: the guard options object is + // shared configuration and is never written (it used to be — `_guard.KnownIdSet = false` — which leaked + // one store's capability into every other consumer of the same options). + private volatile bool _listingUnsupported; + + /// The known-id set and the instant it was loaded, published as one unit. + /// The ids the inner store holds. + /// When they were read. + private sealed record KnownIdSnapshot(IReadOnlySet Ids, DateTimeOffset StampedUtc); + + // R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards + // the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the + // configured rate instead of one SQL round trip per attempt under the SessionManager lock. + private readonly object _bucketGate = new(); + private double _missTokens; + private DateTimeOffset _missRefilledUtc; /// Initializes a new instance of the class. /// The store being bounded. @@ -66,11 +93,13 @@ public sealed class BoundedApiKeyStore : IApiKeyStore /// DataPlaneAuthRegistration for why a second clock here would be a hazard rather than a /// detail. /// + /// The pre-authentication guards (R03:3.3); defaults when omitted. public BoundedApiKeyStore( IApiKeyStore inner, IOptions options, ILogger log, - TimeProvider time) + TimeProvider time, + ApiKeyStoreGuardOptions? guard = null) { ArgumentNullException.ThrowIfNull(inner); ArgumentNullException.ThrowIfNull(options); @@ -81,8 +110,14 @@ public sealed class BoundedApiKeyStore : IApiKeyStore _options = options; _log = log; _time = time; + _guard = guard ?? new ApiKeyStoreGuardOptions(); + _missTokens = Math.Max(0, _guard.MissBurst); + _missRefilledUtc = time.GetUtcNow(); } + /// The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards. + public IReadOnlySet? KnownKeyIds => _knownIds?.Ids; + /// The store being bounded. /// /// Exposed so the composition root's wiring can be asserted. A decorator registered in one branch @@ -115,12 +150,103 @@ public sealed class BoundedApiKeyStore : IApiKeyStore return cached; } + // R03:3.3, guard 1 — an id the ConfigDb does not hold is refused in memory; the store is never asked. + var known = await KnownIdsAsync(ttl, ct).ConfigureAwait(false); + if (known is not null && !known.Contains(apiKeyId)) + { + OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair("outcome", "unknown_id")); + return null; + } + + // R03:3.3, guard 2 — a lookup that will reach the store needs a miss token unless the id is known + // to exist (a known id is a real key being verified, never a guess). + if (known is null && !TryTakeMissToken()) + { + OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair("outcome", "miss_budget")); + throw new InvalidOperationException( + "ApiKeyStore: the unknown-key miss budget is exhausted (Security:ApiKeys:MissBurst / " + + "MissRefillPerSecond); refusing this lookup without a store read."); + } + var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct) .ConfigureAwait(false); + if (fresh is not null && known is null) RefundMissToken(); if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh); return fresh; } + /// + /// The known-id set, refreshed when older than . Single-flight: concurrent + /// callers during a refresh use the previous set (or, before the first load, the per-id path). A + /// refresh that fails or exceeds the deadline logs and keeps the previous set. + /// + /// The refresh interval (the cache TTL); non-positive disables the set. + /// The caller's token. + /// The set, or null. + private async Task?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct) + { + if (!_guard.KnownIdSet || _listingUnsupported || ttl <= TimeSpan.Zero) return null; + + var now = _time.GetUtcNow(); + var current = _knownIds; + if (current is not null && now - current.StampedUtc < ttl) return current.Ids; + + if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current?.Ids; + try + { + // Re-check under the gate: another caller may have refreshed while this one waited. + if (_knownIds is { } refreshed && now - refreshed.StampedUtc < ttl) return refreshed.Ids; + var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false); + if (ids is null) + { + // Unsupported by the inner store: stop asking — for this decorator only. + _listingUnsupported = true; + return null; + } + _knownIds = new KnownIdSnapshot(ids, _time.GetUtcNow()); + return ids; + } + catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) + { + _log.LogWarning(ex, + "ApiKeyStore: refreshing the known key-id set failed; {Mode}", + current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force"); + return current?.Ids; + } + finally + { + _knownIdsRefresh.Release(); + } + } + + /// Takes one miss token, refilling by elapsed time first; always succeeds when the budget is disabled. + /// when the budget is exhausted. + private bool TryTakeMissToken() + { + var burst = _guard.MissBurst; + if (burst <= 0) return true; + lock (_bucketGate) + { + var now = _time.GetUtcNow(); + var elapsed = (now - _missRefilledUtc).TotalSeconds; + if (elapsed > 0) + { + _missTokens = Math.Min(burst, _missTokens + elapsed * Math.Max(0, _guard.MissRefillPerSecond)); + _missRefilledUtc = now; + } + if (_missTokens < 1) return false; + _missTokens -= 1; + return true; + } + } + + /// A lookup that reached the store and HIT was not a miss: its token goes back. + private void RefundMissToken() + { + if (_guard.MissBurst <= 0) return; + lock (_bucketGate) _missTokens = Math.Min(_guard.MissBurst, _missTokens + 1); + } + // Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect. /// public async Task> FindGrantsAsync(string principal, CancellationToken ct) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs index c3f9c1f5..52fcbe51 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs @@ -125,6 +125,16 @@ public sealed class ConfigDbApiKeyStore : IApiKeyStore return await LoadGrantsAsync(db, principal, ct).ConfigureAwait(false); } + /// + public async Task?> ListKeyIdsAsync(CancellationToken ct) + { + await using var db = await _factory.CreateDbContextAsync(ct).ConfigureAwait(false); + // One projection over the whole (small) table — the unique index on ApiKeyId makes this an index + // scan — so an unknown id can be refused in memory instead of costing a query per attempt. + var ids = await db.ApiKeys.AsNoTracking().Select(k => k.ApiKeyId).ToListAsync(ct).ConfigureAwait(false); + return ids.ToHashSet(StringComparer.Ordinal); + } + /// Loads a principal's grants, confined to this node's cluster. /// /// The full grant set, not just the Cluster-scoped rows. The old diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs index c2f82dae..93057153 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs @@ -35,6 +35,16 @@ public interface IApiKeyStore /// The cancellation token. /// The matching ACL rows; empty when none. Task> FindGrantsAsync(string principal, CancellationToken ct); + + /// + /// Every ApiKeyId the store holds (enabled or not), for the pre-authentication known-id set + /// (R03:3.3); when the store cannot cheaply list them, in which case callers + /// fall back to a per-id lookup. The set only answers "does this id exist at all" — enabled, expiry + /// and the secret are still decided by . + /// + /// The cancellation token. + /// The id set, or null when unsupported. + Task?> ListKeyIdsAsync(CancellationToken ct) => Task.FromResult?>(null); } /// An API key resolved for authentication, with the ACL grants naming its principal. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs new file mode 100644 index 00000000..c2377ea1 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs @@ -0,0 +1,73 @@ +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// Lifecycle of this node's OPC UA endpoint as drives it. +public enum OpcUaEndpointStatus +{ + /// The hosted service has not run StartAsync yet. + NotStarted, + + /// The SDK is listening and the address-space / ServiceLevel sinks are bound. + Started, + + /// The SDK failed to start (port in use, PKI unwritable, bad OpcUa:* override, …). + Failed, + + /// The hosted service stopped the SDK; the port is released. + Stopped, +} + +/// +/// The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed +/// start was one Error line: /health/ready stayed 200, the node joined its mesh, could be +/// elected driver Primary and acked deploys as Applied through the Null sink — while clients got +/// connection-refused from the node the fleet called healthy. +/// +/// +/// Consumed by OpcUaEndpointHealthCheck (Ready + Active tiers). The redundancy election and the +/// ServiceLevel calculator do not read it yet — they live outside the Host (Cluster / +/// Runtime); feeding them is the Wave 3 follow-up the lane report names. Until then a fused +/// node whose SDK failed is visible (503) but still Primary-eligible. +/// +public interface IOpcUaEndpointState +{ + /// The endpoint's current lifecycle status. + OpcUaEndpointStatus Status { get; } + + /// Why the start failed, when is . + string? FailureReason { get; } +} + +/// Mutable ; written by the hosted service, read by the health check. +public sealed class OpcUaEndpointState : IOpcUaEndpointState +{ + private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted; + private volatile string? _failureReason; + + /// + public OpcUaEndpointStatus Status => _status; + + /// + public string? FailureReason => _failureReason; + + /// Records a successful SDK start. + public void MarkStarted() + { + _failureReason = null; + _status = OpcUaEndpointStatus.Started; + } + + /// Records a failed SDK start. + /// The failure, as the health probe should report it. + public void MarkFailed(string reason) + { + _failureReason = reason; + _status = OpcUaEndpointStatus.Failed; + } + + /// Records that the hosted service stopped the SDK. + public void MarkStopped() + { + _failureReason = null; + _status = OpcUaEndpointStatus.Stopped; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs index 79f62cd8..609b0087 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Authorization; @@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl private readonly IDriverPrimaryRoleSource? _primaryRoleSource; private readonly HistorianUpdaterHolder _historianUpdater; private readonly IHistoryUpdateRoleView? _historyUpdateRoleView; + private readonly OpcUaEndpointState _endpointState; + private readonly bool _rethrowStartFailure; private OpcUaApplicationHost? _appHost; private OtOpcUaSdkServer? _server; @@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl /// The HistoryUpdate verdict DriverHostActor publishes (the /// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every /// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow). + /// The endpoint lifecycle fact this service maintains for the + /// opcua-endpoint health probe (R03:2.1). Optional: a private instance is kept when none is registered, + /// so a harness that never probes health is unchanged. + /// The node's resolved roles. A node with the driver role and NO admin role exists only + /// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and + /// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through + /// the health probe instead. Null (harnesses) behaves as fused. public OtOpcUaServerHostedService( IOptions options, DeferredAddressSpaceSink deferredSink, @@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl ILoggerFactory loggerFactory, IDriverPrimaryRoleSource? primaryRoleSource = null, HistorianUpdaterHolder? historianUpdater = null, - IHistoryUpdateRoleView? historyUpdateRoleView = null) + IHistoryUpdateRoleView? historyUpdateRoleView = null, + OpcUaEndpointState? endpointState = null, + EffectiveRoles? roles = null) { ArgumentNullException.ThrowIfNull(aclOptions); ArgumentNullException.ThrowIfNull(time); @@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl _primaryRoleSource = primaryRoleSource; _historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None; _historyUpdateRoleView = historyUpdateRoleView; + _endpointState = endpointState ?? new OpcUaEndpointState(); + _rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false }; _logger = loggerFactory.CreateLogger(); } @@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl } catch (Exception ex) { + // R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy + // (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits + // so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct + // the OpcUa:* config without a process bounce — but it is visibly not ready. + _endpointState.MarkFailed(ex.Message); + await ReleaseFailedStartAsync().ConfigureAwait(false); + if (_rethrowStartFailure) + { + _logger.LogCritical(ex, + "OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " + + "node justifies staying up; failing host start so the supervisor restarts the process"); + throw; + } _logger.LogError(ex, - "OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op"); - // Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot. - // Operators see the failure via the logs + can correct config without a process bounce - // of the whole binary. + "OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " + + "OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section"); return; } @@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl { _logger.LogWarning( "OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null"); + _endpointState.MarkFailed("SDK reported started but the node manager is null"); return; } @@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl { _logger.LogError(ex, "OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues"); + _endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message); try { - await StopAsync(cancellationToken).ConfigureAwait(false); + DetachSinks(); } catch (Exception cleanupError) { _logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure"); } - var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync( - async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false), - () => _appHost.DisposeAsync(), _logger).ConfigureAwait(false); - if (released) - { - _appHost = null; - _server = null; - } + await ReleaseEndpointAsync().ConfigureAwait(false); return; } @@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // attached, so the first tick cannot land on a half-wired server. StartRevalidationTimer(); + _endpointState.MarkStarted(); _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound"); } + /// + /// Releases an SDK whose StartAsync threw. The application instance may hold a half-opened + /// listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the + /// transport is up); disposing it is the only way to be sure. The state stays Failed. + /// + private async Task ReleaseFailedStartAsync() + { + var appHost = _appHost; + _appHost = null; + _server = null; + if (appHost is null) return; + try + { + await appHost.DisposeAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw"); + } + } + + /// + /// Stops the SDK and disposes the application host, releasing endpoint ownership only when both + /// succeeded (): on a failure the + /// fields are retained so retries at host disposal. + /// + private async Task ReleaseEndpointAsync() + { + var server = _server; + var appHost = _appHost; + if (server is null && appHost is null) return; + var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync( + async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); }, + () => appHost?.DisposeAsync() ?? ValueTask.CompletedTask, + _logger).ConfigureAwait(false); + if (released) + { + _appHost = null; + _server = null; + } + } + /// /// The ServerHistorian:HonorReturnBounds kill switch StartAsync applies to the node manager /// (default true when the section or key is absent). Exposed so the wiring is testable without booting the SDK. @@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl } /// - /// Stops the OPC UA server asynchronously. + /// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes. /// + /// + /// R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown, + /// the other hosted services' StopAsync), serving the last-written values with an emptied authority: + /// reads succeeded on stale data while writes got "unavailable", and clients saw a server that was + /// up but inert instead of the connection-refused that triggers their failover. + /// stays idempotent with this: ownership is nulled once the stop succeeded. + /// /// Cancellation token. /// A task that represents the asynchronous operation. - public Task StopAsync(CancellationToken cancellationToken) + public async Task StopAsync(CancellationToken cancellationToken) + { + DetachSinks(); + await ReleaseEndpointAsync().ConfigureAwait(false); + if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped(); + } + + /// + /// Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor + /// or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running + /// until . + /// + private void DetachSinks() { // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose // authority and gate have already been cleared is the shape this whole teardown block exists to @@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // request rather than let the seams fall back to their role checks — a stopping node should shed // enforcement in the same order it acquired it. _server?.SetDataPlaneAuthorizer(null); - return Task.CompletedTask; } /// @@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl public async ValueTask DisposeAsync() { // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must - // not leave a tick armed against a server that is about to go away. + // not leave a tick armed against a server that is about to go away. A StopAsync that released the + // endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left + // them set, and this is the retry. _revalidationTimer?.Dispose(); _revalidationTimer = null; - if (_appHost is not null) await _appHost.DisposeAsync(); + var appHost = _appHost; + _appHost = null; + _server = null; + if (appHost is not null) await appHost.DisposeAsync(); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 6404504e..b2300ab6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -162,10 +162,15 @@ builder.AddZbSerilog(o => o.ServiceName = "otopcua"); // central over the fetch-and-cache path (ConfigSource:Mode=FetchAndCache). Registering it // unconditionally would throw on a driver-only node, since AddOtOpcUaConfigDb requires the string. // A fused admin+driver node keeps ConfigDb via its admin role — byte-for-byte unaffected. -// bound unconditionally: a driver node needs the pepper to VERIFY keys, an admin node needs it -// to MINT them, and getting this wrong on either side is a silent authentication failure rather than a -// startup error. -builder.Services.Configure(builder.Configuration.GetSection(ApiKeyOptions.SectionName)); +// A driver node needs the pepper to VERIFY keys, an admin node needs it to MINT them, and getting this +// wrong on either side used to be a silent authentication failure rather than a startup error (R03:2.3): +// ApiKeyOptionsValidator now fails host start on a blank or placeholder pepper wherever either role is +// present. A node with neither role (dev-only) has no key path and keeps the plain bind. +if (hasDriver || hasAdmin) + builder.Services.AddValidatedOptions( + builder.Configuration, ApiKeyOptions.SectionName); +else + builder.Services.Configure(builder.Configuration.GetSection(ApiKeyOptions.SectionName)); // Security:Acl:Enabled. Default false; see AclOptions for why the flip must be a config // change rather than a redeploy, and what it swaps rather than adds. builder.Services.Configure(builder.Configuration.GetSection(AclOptions.SectionName)); @@ -452,6 +457,11 @@ if (hasDriver) // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once. builder.Services.TryAddSingleton>(sp => () => sp.GetRequiredService()); + // The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint + // health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the + // interface, so the writer and the probe cannot disagree. + builder.Services.AddSingleton(); + builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); } @@ -554,8 +564,9 @@ if (hasAdmin) builder.Services.AddOtOpcUaSecrets(builder.Configuration); // hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so -// probing it would throw when the readiness endpoint resolves the factory. -builder.Services.AddOtOpcUaHealth(hasAdmin); +// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the +// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green. +builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver); builder.Services.AddOtOpcUaObservability(builder.Configuration); // gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role, diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs index 9c7c741f..5466ab49 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs @@ -42,8 +42,10 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource // publish. private const uint StatusNeverPushed = OpcUaStatus.BadWaitingForInitialData; - // Intentionally never pruned: cold-start semantics depend on retained last values so - // ReadTag can answer synchronously for any path that has ever been pushed. + // Retained last values: ReadTag answers synchronously for any path that has been pushed. Pruned + // ONLY through RetainOnly — the host calls it with the loaded dependency-ref union after every + // (re)load, so a renamed/removed raw path (a RawTree rename is a surgical remove+add) stops + // costing a boxed value + two DateTimes for the process lifetime (R02:L3). private readonly ConcurrentDictionary _cache = new(StringComparer.Ordinal); private readonly ConcurrentDictionary>> _observers @@ -91,6 +93,28 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource } } + /// + /// Drop every cached snapshot whose path is not in . Observers are + /// untouched. Called by the host after each successful engine (re)load with the union of the loaded + /// alarms' dependency refs — exactly the set the mux will push from now on — so paths no alarm + /// references any more stop being retained. + /// + /// The paths to keep. + /// The number of cached snapshots dropped. + public int RetainOnly(IReadOnlySet paths) + { + ArgumentNullException.ThrowIfNull(paths); + var dropped = 0; + foreach (var key in _cache.Keys) + { + if (!paths.Contains(key) && _cache.TryRemove(key, out _)) dropped++; + } + return dropped; + } + + /// Test/diagnostic: the number of paths currently retained. + public int RetainedPathCount => _cache.Count; + /// public DataValueSnapshot ReadTag(string path) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs index ffa5dc4f..210a84ca 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs @@ -23,10 +23,16 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; /// cluster alerts DistributedPubSub topic. /// /// -/// The host is the engine's lifecycle owner: the caller (the driver-role startup) builds -/// the , constructs a -/// around that same upstream, and passes both here. The host disposes the engine in -/// . +/// The host is the engine's lifecycle owner. The restart-safe shape (R02:H1) is the +/// factory : +/// every incarnation builds its own + a +/// around it, and disposes them in . +/// Akka's default restart runs PostStop then re-creates the actor from the SAME Props, so the +/// legacy instance-taking Props — a fixed engine captured in the Props — restarted around a +/// disposed engine and every later LoadAsync threw: the node's scripted alarms +/// silently died. The legacy overload is kept source-compatible but is restart-safe only in the +/// weaker sense that a restart now DETACHES rather than disposes its engine (see +/// ); production should spawn through the factory. /// /// /// @@ -99,6 +105,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor /// Warning on every clean shutdown. private sealed record AlarmsLoadCanceled; + /// Default bound on one inbound alarm command's engine op (): 10 s. + public static readonly TimeSpan DefaultCommandTimeout = TimeSpan.FromSeconds(10); + private readonly IActorRef _publishActor; private readonly IActorRef? _mux; private readonly DependencyMuxTagUpstreamSource _upstream; @@ -107,6 +116,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor private readonly ILoggingAdapter _log = Context.GetLogger(); private readonly CancellationTokenSource _cts = new(); private readonly EventHandler _onEngineEvent; + private readonly TimeSpan _commandTimeout; + + /// True when THIS incarnation built its engine (factory Props) and therefore disposes it on + /// every stop, restart included. False for the legacy instance-taking Props: the engine outlives the + /// incarnation, so a restart only detaches from it and a final stop disposes it. + private readonly bool _ownsEngine; + + /// Set by so the it triggers can tell a restart + /// (the next incarnation reuses a legacy engine) from a genuine stop. + private bool _restarting; /// Cached local from the latest /// snapshot (null = unknown until the first snapshot arrives, or no wired). The @@ -152,6 +171,8 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor /// Optional node-local live-telemetry hub (Phase 5) each cluster-wide /// alerts transition is also emitted into; null (tests) skips the emit. /// The used to instantiate the actor. + /// Legacy, instance-taking shape — see the class remarks: a supervision restart re-creates the + /// actor around these SAME instances, so prefer the factory overload in production. public static Props Props( IActorRef publishActor, IActorRef? mux, @@ -162,6 +183,60 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor ITelemetryLocalHub? telemetryHub = null) => Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider, telemetryHub)); + /// + /// Restart-safe factory Props (R02:H1): each incarnation of the actor calls + /// in its constructor and owns what it returns — the engine must + /// be constructed around the returned upstream — so a supervision restart gets a fresh, + /// undisposed engine and the next loads cleanly. + /// + /// The OPC UA publish actor emissions are bridged to. + /// Optional dependency multiplexer the host registers interest with. + /// Builds a fresh (upstream, engine) pair; called once per incarnation. + /// The local cluster node id for the Primary alerts-emit gate; null ⇒ default-emit. + /// Test seam for the alerts-emit gate's driver-member count. + /// Optional node-local live-telemetry hub. + /// Bound on one inbound alarm command's engine op; null ⇒ . + /// The used to instantiate the actor. + public static Props Props( + IActorRef publishActor, + IActorRef? mux, + Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory, + NodeId? localNode = null, + Func? driverMemberCountProvider = null, + ITelemetryLocalHub? telemetryHub = null, + TimeSpan? commandTimeout = null) => + Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, engineFactory, localNode, driverMemberCountProvider, telemetryHub, commandTimeout)); + + /// Factory-built incarnation: owns (and on every stop disposes) the engine it builds. + /// The OPC UA publish actor emissions are bridged to. + /// Optional dependency multiplexer the host registers interest with. + /// Builds this incarnation's (upstream, engine) pair. + /// The local cluster node id for the Primary alerts-emit gate. + /// Test seam for the alerts-emit gate's driver-member count. + /// Optional node-local live-telemetry hub. + /// Bound on one inbound alarm command's engine op. + public ScriptedAlarmHostActor( + IActorRef publishActor, + IActorRef? mux, + Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory, + NodeId? localNode = null, + Func? driverMemberCountProvider = null, + ITelemetryLocalHub? telemetryHub = null, + TimeSpan? commandTimeout = null) + : this(publishActor, mux, Build(engineFactory), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: true, commandTimeout) + { + } + + private static (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) Build( + Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory) + { + ArgumentNullException.ThrowIfNull(engineFactory); + var pair = engineFactory(); + ArgumentNullException.ThrowIfNull(pair.Upstream, "engineFactory().Upstream"); + ArgumentNullException.ThrowIfNull(pair.Engine, "engineFactory().Engine"); + return pair; + } + /// Initializes a new instance of the class. /// The OPC UA publish actor emissions are bridged to. /// Optional dependency multiplexer the host registers dependency interest with. @@ -183,14 +258,29 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor NodeId? localNode = null, Func? driverMemberCountProvider = null, ITelemetryLocalHub? telemetryHub = null) + : this(publishActor, mux, (upstream, engine), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: false, commandTimeout: null) + { + } + + private ScriptedAlarmHostActor( + IActorRef publishActor, + IActorRef? mux, + (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) pair, + NodeId? localNode, + Func? driverMemberCountProvider, + ITelemetryLocalHub? telemetryHub, + bool ownsEngine, + TimeSpan? commandTimeout) { ArgumentNullException.ThrowIfNull(publishActor); - ArgumentNullException.ThrowIfNull(upstream); - ArgumentNullException.ThrowIfNull(engine); + ArgumentNullException.ThrowIfNull(pair.Upstream); + ArgumentNullException.ThrowIfNull(pair.Engine); _publishActor = publishActor; _mux = mux; - _upstream = upstream; - _engine = engine; + _upstream = pair.Upstream; + _engine = pair.Engine; + _ownsEngine = ownsEngine; + _commandTimeout = commandTimeout is { } t && t > TimeSpan.Zero ? t : DefaultCommandTimeout; _localNode = localNode; _driverMemberCountProvider = driverMemberCountProvider; _telemetryHub = telemetryHub; @@ -294,8 +384,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor // subscriptions exist, so any DependencyValueChanged the mux delivers will be observed by the // engine. RegisterInterest replaces a subscriber's prior interest set (see DependencyMuxActor), // so a re-Apply with a fresh union simply supersedes the old one — no explicit unregister needed. - _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self)); - _log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs", msg.DepRefs.Count); + // ReplayRetained: the mux replays the last value it saw for each ref, so a fresh engine (a + // supervision restart builds one around an EMPTY upstream) converges immediately instead of + // waiting for every input to change again. + _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self, ReplayRetained: true)); + // Prune the upstream's retained snapshots to the loaded set (R02:L3): a path no alarm references + // any more will never be pushed again, so its last value is dead weight for the process lifetime. + var dropped = _upstream.RetainOnly(new HashSet(msg.DepRefs, StringComparer.Ordinal)); + _log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs (pruned {Dropped} retained path(s))", + msg.DepRefs.Count, dropped); ReassertConditionNodes(); } @@ -591,18 +688,25 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor return; } + // Bounded (R02:H3): every op ends in IAlarmStateStore.SaveAsync (LocalDb/SQLite), and this is a + // ReceiveAsync — the mailbox is suspended until the op returns, so an unbounded await on a hung + // file lock stalled dependency-value ingestion for the duration. Linked to the actor's lifetime + // token so a stop still cancels promptly. + using var cts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token); + cts.CancelAfter(_commandTimeout); + var ct = cts.Token; try { switch (cmd.Operation) { case "Acknowledge": - await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None); + await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct); break; case "Confirm": - await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None); + await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct); break; case "OneShotShelve": - await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, ct); break; case "TimedShelve": // A timed shelve needs the absolute unshelve instant. T18 derives it from the OPC UA @@ -614,16 +718,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor cmd.AlarmId); return; } - await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, CancellationToken.None); + await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, ct); break; case "Unshelve": - await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, ct); break; case "Enable": - await _engine.EnableAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.EnableAsync(cmd.AlarmId, cmd.User, ct); break; case "Disable": - await _engine.DisableAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.DisableAsync(cmd.AlarmId, cmd.User, ct); break; case "AddComment": if (string.IsNullOrWhiteSpace(cmd.Comment)) @@ -631,7 +735,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor _log.Warning("ScriptedAlarmHost: rejecting AddComment for {AlarmId} — empty comment text", cmd.AlarmId); return; } - await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None); + await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct); break; default: _log.Warning("ScriptedAlarmHost: ignoring AlarmCommand with unknown operation {Op} for {AlarmId}", @@ -639,6 +743,11 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor break; } } + catch (OperationCanceledException) when (!_cts.IsCancellationRequested) + { + _log.Warning("ScriptedAlarmHost: engine op {Op} for alarm {AlarmId} timed out after {Timeout} — the condition-state store is stalled", + cmd.Operation, cmd.AlarmId, _commandTimeout); + } catch (Exception ex) { // A failing engine op must not crash the actor — mirror OnLoadFailed's log-and-stay-inert style. @@ -672,16 +781,28 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor base.PreStart(); } + /// + /// Akka's default PreRestart stops the children and calls . The flag + /// lets that keep a legacy (Props-captured) engine alive for the next incarnation + /// instead of disposing the instance it is about to be re-created around. + protected override void PreRestart(Exception reason, object message) + { + _restarting = true; + base.PreRestart(reason, message); + } + /// protected override void PostStop() { // Unregister mux interest first so no further DependencyValueChanged arrives while we tear down. _mux?.Tell(new DependencyMuxActor.UnregisterInterest(Self)); // Cancel any in-flight LoadAsync, detach the OnEvent handler (so a late engine emission can't - // Tell a stopping actor), then dispose the engine (which drains its background work + clears). + // Tell a stopping actor), then dispose the engine (which drains its background work + clears) — + // unless this is a RESTART around a legacy Props-captured engine, which the next incarnation will + // re-attach to (R02:H1). _cts.Cancel(); _engine.OnEvent -= _onEngineEvent; - _engine.Dispose(); + if (_ownsEngine || !_restarting) _engine.Dispose(); _cts.Dispose(); base.PostStop(); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs index 7011492d..b8dc818f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs @@ -62,7 +62,10 @@ public sealed class DependencyConsumerMuxAdapter : ReceiveActor var refs = _consumer.DependencyRefs.ToList(); // A RegisterInterest replaces the prior interest set for this subscriber (mux is idempotent on // re-register), so re-registering after an apply cleanly adopts an added/removed ref. - _mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self)); + // ReplayRetained (R07:H6): a driver Restart is a NEW consumer behind a new adapter; the mux replays + // the last value of every ref it retained so the calc tags compute from current inputs at once + // instead of sitting BadWaitingForInitialData until each input happens to change. + _mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self, ReplayRetained: true)); _log.Debug("DependencyConsumerMuxAdapter: registered {Count} dependency ref(s)", refs.Count); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs index fe57da4e..40e9bf19 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs @@ -26,32 +26,134 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; /// receive as well, for direct senders and tests. /// The mux is deliberately not a DPS subscriber — virtual-tag evaluation is local to each node and /// would over-emit if it spanned the cluster. +/// +/// +/// Last-value retention + replay (R07:H6, bounded in review fix round 1 / I2). The mux +/// retains the last (value, status, timestamp) of a ref only while at least one subscriber +/// registered with names it (the virtual-tag +/// actors, the Calculation mux adapter, the scripted-alarm host). A registration that asks receives +/// one batch of the retained values for its refs, in the shape a driver burst arrives in. The case +/// this serves is a subscriber that goes away and comes back: a Calculation driver Restart is a new +/// driver instance + a new mux adapter, and before this every calc tag sat +/// BadWaitingForInitialData until each of its inputs happened to change again. So when a +/// ref's last replay-opting subscriber leaves, its value is kept for +/// (default ; a returning replay registration inside it cancels +/// the release), then dropped together with the ref's entry. +/// +/// +/// A subscriber that does NOT opt in retains nothing: the continuous-historization recorder +/// registers every historized ref on the node (a replay there would record a stale sample per tag +/// per deploy), and retaining on its behalf made the map grow with the historized address space and +/// never shrink. Memory is therefore bounded by the refs replay-opting subscribers currently name, +/// plus those released within the last grace window. A ref with no subscriber and nothing retained +/// has no entry at all. +/// +/// +/// Source removal. The mux records which driver instance last published each retained ref; +/// evicts every value that driver produced, so a removed driver's last +/// Good is never replayed as current. A comms-loss fan (a Bad batch) already overwrites the retained +/// value with Bad. ⚠️ As of fix round 1 no production actor sends yet — +/// DriverHostActor (outside the scripting lane) must Tell it when it stops / removes a driver +/// child. Until then a removed driver's last value can still be replayed to a replay-opting +/// subscriber that re-registers for that ref while another replay-opting subscriber holds it, or +/// within the grace window. +/// /// -public sealed class DependencyMuxActor : ReceiveActor +public sealed class DependencyMuxActor : ReceiveActor, IWithTimers { public const string ActorName = "dependency-mux"; + /// How long a ref's retained value outlives its last replay-opting subscriber. + public static readonly TimeSpan DefaultRetentionGrace = TimeSpan.FromMinutes(2); + /// Register a subscriber's interest in a set of tag refs. Idempotent on re-register — /// the second call replaces the prior interest set for that subscriber. - public sealed record RegisterInterest(IReadOnlyList TagRefs, IActorRef Subscriber); + /// The refs the subscriber wants. + /// The subscriber. + /// When true, the mux retains the last value of every ref in + /// while this subscriber is registered, and immediately sends the subscriber one + /// holding the last retained value of every ref it has + /// one for (nothing when it has none). + public sealed record RegisterInterest(IReadOnlyList TagRefs, IActorRef Subscriber, bool ReplayRetained = false); /// Unregister every interest held by . Sent on PostStop by /// the subscriber, or by Terminated handling when the mux watches. public sealed record UnregisterInterest(IActorRef Subscriber); - private readonly Dictionary> _byRef = new(StringComparer.Ordinal); - private readonly Dictionary> _bySubscriber = new(); + /// A driver instance was removed / stopped: evict every retained value it produced so it is + /// never replayed. Subscriptions are untouched. + /// The removed driver instance. + public sealed record SourceRemoved(string DriverInstanceId); + + private sealed class SweepReleased + { + public static readonly SweepReleased Instance = new(); + } + + private const string SweepTimerKey = "sweep-released"; + + /// Per-ref state. One lookup on the hot path answers both "who wants it" and "retain it?". + private sealed class RefEntry + { + public readonly HashSet Subscribers = new(); + + /// Registered subscribers of this ref that opted into replay. + public int Replayers; + + /// When the last replay-opting subscriber left (retention in grace), or null. + public DateTime? ReleasedAtUtc; + + public VirtualTagActor.DependencyValueChanged? Last; + public string? LastSource; + + public bool Retains => Replayers > 0 || ReleasedAtUtc is not null; + public bool IsDead => Subscribers.Count == 0 && Replayers == 0 && Last is null; + + public void Forget() + { + Last = null; + LastSource = null; + ReleasedAtUtc = null; + } + } + + private sealed record Registration(HashSet Refs, bool Replay); + + private readonly Dictionary _byRef = new(StringComparer.Ordinal); + private readonly Dictionary _bySubscriber = new(); private readonly ILoggingAdapter _log = Context.GetLogger(); + /// How long a released ref's value is kept. + public TimeSpan RetentionGrace { get; } + + /// + public ITimerScheduler Timers { get; set; } = null!; + + /// Test hook — refs that currently hold a retained value. + internal int RetainedRefCount => _byRef.Values.Count(e => e.Last is not null); + + /// Test hook — refs the mux holds any state for (subscribers or a retained value). + internal int TrackedRefCount => _byRef.Count; + /// Creates props for the DependencyMuxActor. + /// How long a ref's value outlives its last replay-opting subscriber; + /// null ⇒ . /// The props for creating this actor. - public static Props Props() => Akka.Actor.Props.Create(); + public static Props Props(TimeSpan? retentionGrace = null) + { + var grace = retentionGrace ?? DefaultRetentionGrace; + return Akka.Actor.Props.Create(() => new DependencyMuxActor(grace)); + } /// Initializes a new instance of the class. - public DependencyMuxActor() + /// See ; non-positive ⇒ . + public DependencyMuxActor(TimeSpan retentionGrace) { + RetentionGrace = retentionGrace > TimeSpan.Zero ? retentionGrace : DefaultRetentionGrace; Receive(OnRegister); Receive(msg => RemoveSubscriber(msg.Subscriber)); + Receive(OnSourceRemoved); + Receive(_ => OnSweep()); Receive(OnAttributeValuePublished); Receive(OnAttributeValueBatchPublished); Receive(msg => RemoveSubscriber(msg.ActorRef)); @@ -59,60 +161,125 @@ public sealed class DependencyMuxActor : ReceiveActor private void OnRegister(RegisterInterest msg) { - // Replace any prior interest set so re-registers on actor restart don't leak old refs. - if (_bySubscriber.TryGetValue(msg.Subscriber, out var priorRefs)) - { - foreach (var r in priorRefs) - { - if (_byRef.TryGetValue(r, out var set)) - { - set.Remove(msg.Subscriber); - if (set.Count == 0) _byRef.Remove(r); - } - } - } + // Replace any prior interest set so re-registers on actor restart don't leak old refs. Detaching first + // may put a ref into grace; the re-attach below takes it straight back out. + Detach(msg.Subscriber); var refs = new HashSet(msg.TagRefs, StringComparer.Ordinal); - _bySubscriber[msg.Subscriber] = refs; + _bySubscriber[msg.Subscriber] = new Registration(refs, msg.ReplayRetained); + List? replay = null; foreach (var r in refs) { - if (!_byRef.TryGetValue(r, out var set)) + if (!_byRef.TryGetValue(r, out var entry)) { - set = new HashSet(); - _byRef[r] = set; + entry = new RefEntry(); + _byRef[r] = entry; } - set.Add(msg.Subscriber); + entry.Subscribers.Add(msg.Subscriber); + if (!msg.ReplayRetained) continue; + entry.Replayers++; + entry.ReleasedAtUtc = null; + if (entry.Last is { } retained) + (replay ??= []).Add(retained); } Context.Watch(msg.Subscriber); - _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs", msg.Subscriber, refs.Count); + _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs (replay {Replay})", + msg.Subscriber, refs.Count, msg.ReplayRetained); + + // One batch, after the registration is complete, in exactly the shape a driver burst arrives in — + // so the subscriber's hot path handles it with no special case. + if (replay is not null) + { + msg.Subscriber.Tell(new VirtualTagActor.DependencyValueBatch(replay)); + _log.Debug("DependencyMux: replayed {Count} retained value(s) to {Sub}", replay.Count, msg.Subscriber); + } } private void RemoveSubscriber(IActorRef subscriber) { - if (!_bySubscriber.TryGetValue(subscriber, out var refs)) return; - foreach (var r in refs) - { - if (_byRef.TryGetValue(r, out var set)) - { - set.Remove(subscriber); - if (set.Count == 0) _byRef.Remove(r); - } - } - _bySubscriber.Remove(subscriber); + if (!Detach(subscriber)) return; Context.Unwatch(subscriber); _log.Debug("DependencyMux: subscriber {Sub} removed", subscriber); } + /// Drop 's registration. A ref losing its last replay-opting + /// subscriber goes into grace; a ref left with no state at all is removed. + private bool Detach(IActorRef subscriber) + { + if (!_bySubscriber.Remove(subscriber, out var registration)) return false; + var now = DateTime.UtcNow; + var released = false; + foreach (var r in registration.Refs) + { + if (!_byRef.TryGetValue(r, out var entry)) continue; + entry.Subscribers.Remove(subscriber); + if (registration.Replay && --entry.Replayers == 0 && entry.Last is not null) + { + entry.ReleasedAtUtc = now; + released = true; + } + if (entry.IsDead) _byRef.Remove(r); + } + if (released && !Timers.IsTimerActive(SweepTimerKey)) + Timers.StartPeriodicTimer(SweepTimerKey, SweepReleased.Instance, RetentionGrace); + return true; + } + + /// Drop retained values whose grace has run out — with a periodic sweep at the grace interval a + /// released value lives between one and two grace periods — and stop the sweep once nothing is in grace. + private void OnSweep() + { + var cutoff = DateTime.UtcNow - RetentionGrace; + var stillInGrace = false; + List? dead = null; + foreach (var (r, entry) in _byRef) + { + if (entry.ReleasedAtUtc is not { } releasedAt) continue; + if (releasedAt > cutoff) + { + stillInGrace = true; + continue; + } + entry.Forget(); + if (entry.IsDead) (dead ??= []).Add(r); + } + if (dead is not null) + foreach (var r in dead) _byRef.Remove(r); + if (!stillInGrace) Timers.Cancel(SweepTimerKey); + } + + private void OnSourceRemoved(SourceRemoved msg) + { + List? dead = null; + var evicted = 0; + foreach (var (r, entry) in _byRef) + { + if (entry.Last is null || !string.Equals(entry.LastSource, msg.DriverInstanceId, StringComparison.Ordinal)) + continue; + entry.Forget(); + evicted++; + if (entry.IsDead) (dead ??= []).Add(r); + } + if (dead is not null) + foreach (var r in dead) _byRef.Remove(r); + _log.Debug("DependencyMux: source {Driver} removed — evicted {Count} retained value(s)", msg.DriverInstanceId, evicted); + } + private void OnAttributeValuePublished(DriverInstanceActor.AttributeValuePublished msg) { - if (!_byRef.TryGetValue(msg.FullReference, out var subscribers) || subscribers.Count == 0) + if (!_byRef.TryGetValue(msg.FullReference, out var entry)) { - // No virtual tag cares about this ref — drop. Common in normal operation; the address + // Nothing is registered for (or retaining) this ref — drop. Common in normal operation; the address // space carries thousands of tags and only a fraction feed virtual-tag expressions. return; } var dep = new VirtualTagActor.DependencyValueChanged(msg.FullReference, msg.Value, msg.TimestampUtc, msg.StatusCode); - foreach (var sub in subscribers) + if (entry.Retains) + { + entry.Last = dep; + entry.LastSource = msg.DriverInstanceId; + } + foreach (var sub in entry.Subscribers) { // A single-item batch, so the mux emits exactly ONE message shape and every subscriber needs // only the batch receive on its hot path. One list per subscriber: batch ownership transfers @@ -122,18 +289,9 @@ public sealed class DependencyMuxActor : ReceiveActor } } - /// Filters one batched driver production burst per item and delivers each subscriber's matched - /// slice as ONE . Items nobody registered for are - /// dropped (the common case: only a fraction of an address space feeds virtual-tag expressions), so a - /// burst of N values costs one inbound message plus one outbound message per genuinely interested - /// SUBSCRIBER — not per (item, subscriber) pair as it did when this fanned out per value. That is what - /// lets evaluate its script once per burst instead of once per input. - /// The registered set is NOT just virtual-tag inputs: ContinuousHistorizationRecorder - /// registers interest in every historized ref, so on a historizing node can be - /// large and a sizeable fraction of each burst resolves. private void OnAttributeValueBatchPublished(DriverInstanceActor.AttributeValueBatchPublished msg) { - // Nobody has registered anything — skip the whole burst rather than probing an empty map per item. + // Nothing registered and nothing retained — skip the whole burst rather than probing an empty map per item. if (_byRef.Count == 0) return; // Allocated lazily on the first matched item: a burst in which nothing is registered for — still @@ -146,8 +304,15 @@ public sealed class DependencyMuxActor : ReceiveActor for (var i = 0; i < count; i++) { var item = msg.Items[i]; - if (!_byRef.TryGetValue(item.FullReference, out var subscribers) || subscribers.Count == 0) continue; + if (!_byRef.TryGetValue(item.FullReference, out var entry)) continue; var dep = new VirtualTagActor.DependencyValueChanged(item.FullReference, item.Value, item.TimestampUtc, item.StatusCode); + if (entry.Retains) + { + entry.Last = dep; + entry.LastSource = msg.DriverInstanceId; + } + var subscribers = entry.Subscribers; + if (subscribers.Count == 0) continue; // retained (in grace) for a returning subscriber; nobody to forward to now perSubscriber ??= new Dictionary>(); foreach (var sub in subscribers) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs index 3d2a3058..8c7b4f41 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs @@ -7,6 +7,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Observability; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry; namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; @@ -76,6 +77,10 @@ public sealed class VirtualTagActor : ReceiveActor private readonly IReadOnlyList _dependencyRefs; private readonly IActorRef? _mux; private readonly ITelemetryLocalHub? _telemetryHub; + // R02:P6 — a script failing on every trigger of a 1 Hz input published one cluster-wide DPS + // ScriptLogEntry per second per tag; 200 such tags shed everyone's entries out of the bounded + // telemetry channels. The per-failure entry is now rate-limited per tag (20/min + one summary). + private readonly ScriptLogRateLimiter _failureLogLimiter; private readonly ILoggingAdapter _log = Context.GetLogger(); // Dependency SNAPSHOTS, not bare values (archreview 02/S14): the status code is what distinguishes a // comms-lost input from a genuine Good null, and it is what the script reads through @@ -104,6 +109,8 @@ public sealed class VirtualTagActor : ReceiveActor /// Optional reference to a dependency multiplexer actor. /// Optional node-local live-telemetry hub each script-log entry is also /// emitted into (Phase 5); null (tests) simply skips the emit. + /// Rate limiter for this tag's per-failure script-log entries; null ⇒ + /// the default 20 per minute + one summary. /// A configured for creating the actor. public static Props Props( string virtualTagId, @@ -113,7 +120,8 @@ public sealed class VirtualTagActor : ReceiveActor Func? publisherFactory = null, IReadOnlyList? dependencyRefs = null, IActorRef? mux = null, - ITelemetryLocalHub? telemetryHub = null) => + ITelemetryLocalHub? telemetryHub = null, + ScriptLogRateLimiter? failureLogLimiter = null) => Akka.Actor.Props.Create(() => new VirtualTagActor( virtualTagId, expression, evaluator ?? NullVirtualTagEvaluator.Instance, @@ -121,7 +129,8 @@ public sealed class VirtualTagActor : ReceiveActor publisherFactory, dependencyRefs ?? Array.Empty(), mux, - telemetryHub)); + telemetryHub, + failureLogLimiter)); /// Initializes a virtual tag actor with the given configuration and dependencies. /// Unique identifier for the virtual tag. @@ -133,6 +142,8 @@ public sealed class VirtualTagActor : ReceiveActor /// Optional reference to a dependency multiplexer actor. /// Optional node-local live-telemetry hub each script-log entry is also /// emitted into (Phase 5); null (tests) simply skips the emit. + /// Rate limiter for this tag's per-failure script-log entries; null ⇒ + /// the default 20 per minute + one summary. public VirtualTagActor( string virtualTagId, string expression, @@ -141,7 +152,8 @@ public sealed class VirtualTagActor : ReceiveActor Func? publisherFactory, IReadOnlyList dependencyRefs, IActorRef? mux, - ITelemetryLocalHub? telemetryHub = null) + ITelemetryLocalHub? telemetryHub = null, + ScriptLogRateLimiter? failureLogLimiter = null) { _virtualTagId = virtualTagId; _scriptId = scriptId; @@ -151,6 +163,7 @@ public sealed class VirtualTagActor : ReceiveActor _dependencyRefs = dependencyRefs; _mux = mux; _telemetryHub = telemetryHub; + _failureLogLimiter = failureLogLimiter ?? new ScriptLogRateLimiter(); Receive(OnDependencyChanged); Receive(OnDependencyBatch); @@ -162,7 +175,9 @@ public sealed class VirtualTagActor : ReceiveActor { if (_mux is not null && _dependencyRefs.Count > 0) { - _mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self)); + // ReplayRetained: a respawned child (plan edit, supervision restart) evaluates from the mux's + // last-known inputs at once instead of waiting for each to change again. + _mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self, ReplayRetained: true)); } } @@ -353,7 +368,18 @@ public sealed class VirtualTagActor : ReceiveActor private void OnEvaluationFailed(string reason, string level, DateTime timestampUtc) { OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair("outcome", "fail")); - PublishLog(level, reason); + + // Rate-limited per tag (R02:P6): the first 20 failures a minute publish as before; the rest are + // counted and surfaced as ONE summary entry when the window rolls. The metric above and the Bad + // publish below are not limited — only the cluster-wide log fan-out is. + var admission = _failureLogLimiter.Admit(_virtualTagId); + if (admission.SuppressedInPreviousWindow > 0) + { + PublishLog("Warning", + $"{admission.SuppressedInPreviousWindow} further evaluation failure(s) were suppressed from the script log " + + $"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {_failureLogLimiter.LimitPerWindow}/window); last: {reason}"); + } + if (admission.Admitted) PublishLog(level, reason); if ((_hasPublished && _lastPublishedStatus == OpcUaStatus.BadInternalError) || !AllDependenciesArrived()) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs index d00dfb36..f97fa387 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs @@ -3,6 +3,7 @@ using Akka.Event; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Core.VirtualTags; using ZB.MOM.WW.OtOpcUa.OpcUaServer; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; @@ -112,6 +113,11 @@ public sealed class VirtualTagHostActor : ReceiveActor (_evaluator as IScriptCacheOwner)?.ClearCompiledScripts(); } + // Every apply is a deploy, and a deploy lifts the run guard's quarantines unconditionally — "the + // operator redeployed to retry" is one of the two documented ways out of a quarantine (the other, + // an edited expression, the guard detects on its own). Cheap on a no-op redeploy (an empty map). + (_evaluator as IScriptQuarantineOwner)?.ResetQuarantines(); + var desired = new HashSet(msg.Plans.Select(p => p.VirtualTagId), StringComparer.Ordinal); // Stop + forget children whose vtagId is no longer desired. Stopping the child triggers its diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs new file mode 100644 index 00000000..13ede448 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs @@ -0,0 +1,95 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Components.Authorization; +using Microsoft.AspNetCore.Http; +using ZB.MOM.WW.Auth.AspNetCore; + +namespace ZB.MOM.WW.OtOpcUa.Security.Audit; + +/// +/// that reads the Blazor circuit's principal first (R03:2.5). +/// +/// +/// +/// reads IHttpContextAccessor, which is right for a +/// minimal-API request but wrong inside a Blazor Server circuit: after the first render there is no +/// current HttpContext on the circuit's threads, so every audit row written from an interactive +/// page would have been stamped "system". The circuit's identity lives in the scoped +/// , so that is consulted first and the HTTP context is the +/// fallback for non-circuit scopes (where the provider is not registered at all). +/// +/// +/// CurrentActor is synchronous by contract; the server-side provider's +/// is already completed in a +/// circuit, and an incomplete task is treated as "no circuit identity" rather than blocked on. +/// +/// +public sealed class AuthenticationStateAuditActorAccessor : IAuditActorAccessor +{ + private readonly IHttpContextAccessor _httpContextAccessor; + private readonly AuthenticationStateProvider? _authenticationState; + + /// Initializes a new instance of the class. + /// The HTTP context accessor (minimal-API fallback). + /// The circuit's authentication state, when resolved inside a Blazor circuit. + public AuthenticationStateAuditActorAccessor( + IHttpContextAccessor httpContextAccessor, + AuthenticationStateProvider? authenticationState = null) + { + ArgumentNullException.ThrowIfNull(httpContextAccessor); + _httpContextAccessor = httpContextAccessor; + _authenticationState = authenticationState; + } + + /// + public string? CurrentActor + { + get + { + if (_authenticationState is not null) + { + var actor = TryResolveCircuitActor(_authenticationState); + if (actor is not null) return actor; + } + return ResolveActor(_httpContextAccessor.HttpContext?.User); + } + } + + /// + /// The circuit's actor, or null when this scope has no circuit identity. The server-side provider + /// is registered for EVERY scope on an AdminUI node, but only a circuit calls + /// SetAuthenticationState; anywhere else (a minimal-API request, a hosted service's scope) + /// GetAuthenticationStateAsync throws synchronously + /// (fix round 1 / I5). That is "no circuit", never an audit failure — the caller falls back to + /// the HTTP context. + /// + /// The scoped provider. + /// The actor, or null. + private static string? TryResolveCircuitActor(AuthenticationStateProvider provider) + { + Task task; + try + { + task = provider.GetAuthenticationStateAsync(); + } + catch (InvalidOperationException) + { + return null; + } + return task.IsCompletedSuccessfully ? ResolveActor(task.Result.User) : null; + } + + /// The claim-resolution rule shared with . + /// The principal, or null. + /// The actor string, or null when unauthenticated. + internal static string? ResolveActor(ClaimsPrincipal? user) + { + if (user?.Identity?.IsAuthenticated != true) + return null; + + // Prefer the canonical login-name claim; fall back to the Name claim or + // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name). + return user.FindFirst(ZbClaimTypes.Username)?.Value + ?? user.FindFirst(ZbClaimTypes.Name)?.Value + ?? user.Identity.Name; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs index 9a86339f..529504b3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs @@ -1,5 +1,4 @@ using Microsoft.AspNetCore.Http; -using ZB.MOM.WW.Auth.AspNetCore; namespace ZB.MOM.WW.OtOpcUa.Security.Audit; @@ -17,10 +16,10 @@ namespace ZB.MOM.WW.OtOpcUa.Security.Audit; /// , in that order. /// /// -/// Registered as scoped in -/// so that it correctly follows the request scope used by Blazor Server interactive components -/// and minimal-API endpoints. IHttpContextAccessor is registered by -/// AddOtOpcUaAuth via services.AddHttpContextAccessor(). +/// No longer the registered implementation: is, because +/// a Blazor circuit has no HttpContext after its first render and this accessor would stamp every row +/// written from an interactive page "system". Kept as the explicit HTTP-only variant for non-circuit +/// hosts and tests. /// /// public sealed class HttpAuditActorAccessor : IAuditActorAccessor @@ -35,19 +34,6 @@ public sealed class HttpAuditActorAccessor : IAuditActorAccessor } /// - public string? CurrentActor - { - get - { - var user = _httpContextAccessor.HttpContext?.User; - if (user?.Identity?.IsAuthenticated != true) - return null; - - // Prefer the canonical login-name claim; fall back to the Name claim or - // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name). - return user.FindFirst(ZbClaimTypes.Username)?.Value - ?? user.FindFirst(ZbClaimTypes.Name)?.Value - ?? user.Identity.Name; - } - } + public string? CurrentActor => + AuthenticationStateAuditActorAccessor.ResolveActor(_httpContextAccessor.HttpContext?.User); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs new file mode 100644 index 00000000..7c30915e --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs @@ -0,0 +1,39 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; + +namespace ZB.MOM.WW.OtOpcUa.Security.Auth; + +/// The authentication scheme names OtOpcUa registers. +public static class OtOpcUaAuthSchemes +{ + /// + /// The bearer scheme that validates the HS256 tokens /auth/token mints: the first-party + /// AddJwtBearer handler over . + /// Registered only when . + /// + public const string Bearer = "Bearer"; + + /// + /// The default scheme: a policy scheme that forwards to when the request + /// carries an Authorization: Bearer header and to the cookie scheme otherwise, so one + /// RequireAuthorization() / policy accepts a browser session AND a machine token. + /// + public const string CookieOrBearer = "OtOpcUa.CookieOrBearer"; + + /// Whether the request presents a bearer token (the selector's whole rule). + /// The request. + /// when an Authorization: Bearer … header is present. + public static bool HasBearerToken(HttpContext context) => + TryGetBearerToken(context.Request.Headers.Authorization, out _); + + internal static bool TryGetBearerToken(StringValues authorization, out string token) + { + token = string.Empty; + var header = authorization.ToString(); + const string prefix = "Bearer "; + if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; + token = header[prefix.Length..].Trim(); + return token.Length > 0; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs deleted file mode 100644 index 0e42b789..00000000 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System.Net.Http.Json; -using System.Security.Claims; -using Microsoft.AspNetCore.Components.Authorization; -using Microsoft.Extensions.Logging; - -namespace ZB.MOM.WW.OtOpcUa.Security.Blazor; - -/// -/// Blazor Server that snapshots the cookie-backed -/// principal supplied at circuit boot and polls /auth/ping every 60 seconds to detect -/// expiry. Mirrors ScadaLink's CentralUI implementation. -/// -public sealed class CookieAuthenticationStateProvider : AuthenticationStateProvider, IAsyncDisposable -{ - private static readonly TimeSpan PingInterval = TimeSpan.FromSeconds(60); - - private readonly HttpClient _http; - private readonly ILogger _logger; - private readonly CancellationTokenSource _cts = new(); - private ClaimsPrincipal _current; - private Task? _pingLoop; - - /// Initializes a new instance of the CookieAuthenticationStateProvider class. - /// The initial claims principal from circuit boot. - /// The HTTP client for authentication ping requests. - /// The logger for diagnostic messages. - public CookieAuthenticationStateProvider( - ClaimsPrincipal initial, - HttpClient http, - ILogger logger) - { - _current = initial; - _http = http; - _logger = logger; - } - - /// - public override Task GetAuthenticationStateAsync() - { - _pingLoop ??= Task.Run(() => PingLoopAsync(_cts.Token)); - return Task.FromResult(new AuthenticationState(_current)); - } - - private async Task PingLoopAsync(CancellationToken ct) - { - try - { - while (!ct.IsCancellationRequested) - { - await Task.Delay(PingInterval, ct).ConfigureAwait(false); - var resp = await _http.GetAsync("/auth/ping", ct).ConfigureAwait(false); - if (!resp.IsSuccessStatusCode && _current.Identity?.IsAuthenticated == true) - { - _logger.LogInformation("/auth/ping returned {Code}; notifying circuit", (int)resp.StatusCode); - _current = new ClaimsPrincipal(new ClaimsIdentity()); - NotifyAuthenticationStateChanged( - Task.FromResult(new AuthenticationState(_current))); - } - } - } - catch (OperationCanceledException) { /* expected on shutdown */ } - catch (Exception ex) - { - _logger.LogWarning(ex, "Auth ping loop terminated unexpectedly"); - } - } - - /// Cancels the background ping loop, awaits its completion, and disposes the cancellation token source. - /// A task that represents the asynchronous operation. - public async ValueTask DisposeAsync() - { - _cts.Cancel(); - if (_pingLoop is not null) - { - try { await _pingLoop.ConfigureAwait(false); } catch { /* swallow shutdown errors */ } - } - _cts.Dispose(); - } -} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs index 56ac6ef7..a3f661b6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs @@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using ZB.MOM.WW.Auth.AspNetCore; using ZB.MOM.WW.Auth.Abstractions.Roles; using ZB.MOM.WW.OtOpcUa.Security.Jwt; @@ -23,6 +24,17 @@ public static class AuthEndpoints /// Response for a token issue request. public sealed record TokenResponse(string Token); + /// + /// The /auth/token policy: an authenticated principal from the COOKIE scheme. Naming the scheme + /// makes the authorization middleware authenticate that scheme alone, so an Authorization: Bearer + /// header is never consulted here (with login disabled the auto-login handler sits under the same + /// scheme name, so the dev rig is unchanged). + /// + internal static readonly AuthorizationPolicy CookieSessionOnly = + new AuthorizationPolicyBuilder(CookieAuthenticationDefaults.AuthenticationScheme) + .RequireAuthenticatedUser() + .Build(); + /// Maps OtOpcUa authentication endpoints to the application route builder. /// The endpoint route builder. /// The endpoint route builder for chaining. @@ -36,7 +48,17 @@ public static class AuthEndpoints // AddOtOpcUaAuth fallback policy that would otherwise 401 the request. app.MapPost("/auth/login", (Delegate)LoginAsync).AllowAnonymous().DisableAntiforgery(); app.MapGet("/auth/ping", (Delegate)Ping).AllowAnonymous(); - app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(); + // Mapped only when a signing key is configured (JwtOptions.IsConfigured): an unconfigured + // endpoint used to answer HTTP 500 from the JwtTokenService constructor on every call. + if (app.ServiceProvider.GetRequiredService>().Value.IsConfigured) + // COOKIE scheme only (fix round 1 / I2): the default scheme also accepts a bearer, and a bearer + // that could mint a fresh token would renew itself forever — outliving its own expiry and the + // user's LDAP membership. A token is minted only for an interactive (cookie) session. + app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(CookieSessionOnly); + else + app.ServiceProvider.GetService()? + .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") + .LogInformation("Security:Jwt:SigningKey is not configured — /auth/token is not mapped and bearer tokens are not accepted"); app.MapPost("/auth/logout", (Delegate)LogoutAsync).RequireAuthorization(); return app; } @@ -73,8 +95,13 @@ public static class AuthEndpoints { result = await ldap.AuthenticateAsync(username, password, ct); } - catch (Exception) + catch (Exception ex) when (ex is not OperationCanceledException) { + // R03:2.7 — a directory outage used to leave NOTHING in the server log from this path; the + // operator had to infer every 503 from the directory side. + http.RequestServices.GetService()? + .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") + .LogWarning(ex, "LDAP authentication failed for {User}; answering 503", username); return Results.StatusCode(StatusCodes.Status503ServiceUnavailable); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs index 07cbdda4..f523a033 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs @@ -15,4 +15,26 @@ public sealed class JwtOptions /// Default token expiry. Mirrors ScadaLink (15 min). public int ExpiryMinutes { get; set; } = 15; + + /// + /// Whether the JWT feature is on: a non-blank . When off, /auth/token + /// is not mapped and no bearer scheme is registered, so an admin node that never configured a key + /// keeps booting exactly as before; when on, the key is validated at host start + /// (ValidateOnStart) instead of throwing HTTP 500 on the first token request. + /// + public bool IsConfigured => !string.IsNullOrWhiteSpace(SigningKey); + + /// The validation rule shared by the startup validator and the tests: why the options are invalid, or null. + /// A failure message, or when valid. + public string? Validate() + { + if (!IsConfigured) return null; + var keyBytes = System.Text.Encoding.UTF8.GetByteCount(SigningKey); + if (keyBytes < MinSigningKeyBytes) + return $"{SectionName}:SigningKey must be at least {MinSigningKeyBytes} bytes (256 bits) UTF-8 for HMAC-SHA256; the configured key is {keyBytes} byte(s)."; + if (string.IsNullOrWhiteSpace(Issuer)) return $"{SectionName}:Issuer must not be blank."; + if (string.IsNullOrWhiteSpace(Audience)) return $"{SectionName}:Audience must not be blank."; + if (ExpiryMinutes <= 0) return $"{SectionName}:ExpiryMinutes must be > 0."; + return null; + } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs new file mode 100644 index 00000000..c3805267 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.Options; + +namespace ZB.MOM.WW.OtOpcUa.Security.Jwt; + +/// +/// Startup validator for (R03:2.4): when a signing key is configured it must +/// be usable, and the host must learn that at start (ValidateOnStart) rather than from an HTTP 500 +/// on the first /auth/token. An unconfigured section (no key) is valid — the feature is simply off. +/// +public sealed class JwtOptionsValidator : IValidateOptions +{ + /// + public ValidateOptionsResult Validate(string? name, JwtOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var failure = options.Validate(); + return failure is null ? ValidateOptionsResult.Success : ValidateOptionsResult.Fail(failure); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs index 4dd38a2e..9dae8c14 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs @@ -25,24 +25,12 @@ public sealed class JwtTokenService /// /// Role claim type used in the JWT payload. /// - /// Issued-only / no internal JwtBearer scheme: OtOpcUa uses a single Cookie - /// authentication scheme; the JWT is minted by the /auth/token endpoint and - /// consumed externally (e.g. by OPC-UA clients or automation scripts). There is no - /// AddJwtBearer pipeline in OtOpcUa — the cookie stores the - /// directly. Because no internal - /// bearer validation path exists, the short "Role" key is intentionally used here rather - /// than the long URI; external consumers receive exactly the - /// key they expect. - /// - /// - /// If a JwtBearer scheme is ever added: the - /// passed to - /// AddJwtBearer MUST set RoleClaimType = JwtTokenService.RoleClaimType (and - /// NameClaimType = JwtTokenService.UsernameClaimType) so that - /// [Authorize(Roles=...)] and ClaimsPrincipal.IsInRole resolve correctly. - /// is already wired to do this and MUST be used - /// rather than constructing - /// ad hoc. + /// The short "Role" key is what mints and what external consumers of the token + /// expect. The internal bearer scheme (AddJwtBearer in AddOtOpcUaAuth) validates with + /// , which sets RoleClaimType = RoleClaimType and + /// NameClaimType = UsernameClaimType, and turns MapInboundClaims off, so + /// [Authorize(Roles=...)] and ClaimsPrincipal.IsInRole resolve against this key. Never + /// construct ad hoc. /// /// public const string RoleClaimType = "Role"; @@ -150,5 +138,9 @@ public sealed class JwtTokenService // if this is ever passed to AddJwtBearer. See RoleClaimType doc comment for rationale. RoleClaimType = RoleClaimType, NameClaimType = UsernameClaimType, + // Pinned to the one algorithm Issue signs with: an unsigned (alg:none) token or one signed with a + // different HMAC variant over the same key is refused on both the bearer and TryValidate paths. + ValidAlgorithms = [SecurityAlgorithms.HmacSha256], + RequireSignedTokens = true, }; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs index 6df24cad..494145be 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.Cookies; +using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; @@ -19,9 +20,12 @@ using ZB.MOM.WW.OtOpcUa.Security.Ldap; namespace ZB.MOM.WW.OtOpcUa.Security; /// -/// DI registration for OtOpcUa auth. Single Cookie scheme (the JWT lives inside the -/// cookie as its credential payload); no JwtBearer parallel scheme. Matches ScadaBridge -/// structurally — see docs/plans/2026-05-29-auth-alignment-design.md. +/// DI registration for OtOpcUa auth. The cookie scheme carries the browser session; when +/// Security:Jwt:SigningKey is configured a scheme +/// accepts the tokens /auth/token mints (R03:2.4), and the default scheme is a policy scheme +/// that forwards to whichever the request presents — so /api/* and every +/// RequireAuthorization endpoint take a browser session OR a machine token. Matches +/// ScadaBridge structurally — see docs/plans/2026-05-29-auth-alignment-design.md. /// public static class ServiceCollectionExtensions { @@ -36,7 +40,14 @@ public static class ServiceCollectionExtensions /// The same service collection, for chaining. public static IServiceCollection AddOtOpcUaAuth(this IServiceCollection services, IConfiguration configuration) { - services.AddOptions().Bind(configuration.GetSection(JwtOptions.SectionName)); + // JWT is opt-in by key: with no SigningKey the feature is off (no /auth/token, no bearer scheme) + // and an admin node boots as before; with one, the key is validated at host START (R03:2.4) — + // the JwtTokenService constructor used to throw on the first /auth/token call (HTTP 500). + services.AddOptions() + .Bind(configuration.GetSection(JwtOptions.SectionName)) + .ValidateOnStart(); + services.AddSingleton, JwtOptionsValidator>(); + var jwtConfigured = configuration.GetSection(JwtOptions.SectionName).Get()?.IsConfigured ?? false; services.AddOptions().Bind(configuration.GetSection(OtOpcUaCookieOptions.SectionName)); services.AddOptions().Bind(configuration.GetSection(LdapOptions.SectionName)); services.AddOptions().Bind(configuration.GetSection(AuthDisableLoginOptions.SectionName)); @@ -53,7 +64,10 @@ public static class ServiceCollectionExtensions // as the Actor field when constructing a canonical ZB.MOM.WW.Audit.AuditEvent. Registered // Scoped so it correctly follows the request scope used by Blazor Server and minimal-API // endpoints. - services.TryAddScoped(); + // AuthenticationStateAuditActorAccessor reads the Blazor circuit's principal first (no HttpContext + // exists on a circuit's threads after the first render) and falls back to the HTTP context for + // minimal-API scopes, where AuthenticationStateProvider is not registered (R03:2.5). + services.TryAddScoped(); // Singleton — OtOpcUaLdapAuthService is stateless (the shared-library directory client it // wraps opens/disposes an LdapConnection per call) and must be consumable by the Singleton @@ -78,7 +92,39 @@ public static class ServiceCollectionExtensions .GetSection(AuthDisableLoginOptions.SectionName) .GetValue(nameof(AuthDisableLoginOptions.DisableLogin)); - var authBuilder = services.AddAuthentication(CookieAuthenticationDefaults.AuthenticationScheme); + // The default scheme is the cookie-or-bearer selector; sign-in/out stay on the cookie scheme, which is + // the only one that can issue a session. With JWT off the selector always picks the cookie. + var authBuilder = services.AddAuthentication(o => + { + o.DefaultScheme = OtOpcUaAuthSchemes.CookieOrBearer; + o.DefaultSignInScheme = CookieAuthenticationDefaults.AuthenticationScheme; + o.DefaultSignOutScheme = CookieAuthenticationDefaults.AuthenticationScheme; + }); + authBuilder.AddPolicyScheme(OtOpcUaAuthSchemes.CookieOrBearer, "Cookie or Bearer", o => + { + o.ForwardDefaultSelector = ctx => + jwtConfigured && OtOpcUaAuthSchemes.HasBearerToken(ctx) + ? OtOpcUaAuthSchemes.Bearer + : CookieAuthenticationDefaults.AuthenticationScheme; + }); + if (jwtConfigured) + { + // First-party JwtBearer (R03:2.4, fix round 1 / I1) — the earlier hand-rolled handler is gone. + // Validation parameters come from the SAME JwtTokenService that mints, so issuer/audience/key/ + // claim types can never drift; BuildValidationParameters pins ValidAlgorithms = HS256 (an + // alg:none or a different-HMAC token is refused). MapInboundClaims=false keeps the minted + // "Role"/"zb:username" claim keys verbatim, which RoleClaimType/NameClaimType name. A bad token + // is AuthenticateResult.Fail ⇒ 401 + WWW-Authenticate: Bearer, never a 500. + authBuilder.AddJwtBearer(OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { }); + services.AddOptions(OtOpcUaAuthSchemes.Bearer) + .Configure((o, jwt) => + { + o.TokenValidationParameters = jwt.BuildValidationParameters(); + o.MapInboundClaims = false; + o.SaveToken = false; + o.IncludeErrorDetails = false; + }); + } if (disableLogin) { // DEV/TEST ONLY: replace the cookie handler with an always-succeeding handler registered @@ -142,8 +188,8 @@ public static class ServiceCollectionExtensions services.AddAuthorization(o => { - o.FallbackPolicy = new AuthorizationPolicyBuilder( - CookieAuthenticationDefaults.AuthenticationScheme) + // Named on the selector, not the cookie scheme, so the fallback accepts a bearer token too. + o.FallbackPolicy = new AuthorizationPolicyBuilder(OtOpcUaAuthSchemes.CookieOrBearer) .RequireAuthenticatedUser() .Build(); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj index d81e4eb6..8ee03814 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj @@ -10,6 +10,10 @@ + + + + diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs new file mode 100644 index 00000000..aa16ead6 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs @@ -0,0 +1,117 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Tests; + +/// +/// R03:3.2 on the real provider: the migrated config database allows SNAPSHOT isolation (migration +/// EnableSnapshotIsolation) and reads under +/// it. The InMemory tests cannot see either — the provider has no transactions — so this is the one +/// place the deploy path's "one consistent read" is proven against SQL Server. +/// +[Trait("Category", "SchemaCompliance")] +[Collection(nameof(SchemaComplianceCollection))] +public sealed class DraftSnapshotFactorySqlServerTests +{ + private readonly SchemaComplianceFixture _fixture; + + /// Initializes a new instance of the class. + /// The migrated test database. + public DraftSnapshotFactorySqlServerTests(SchemaComplianceFixture fixture) => _fixture = fixture; + + /// The migration leaves the database with snapshot isolation allowed. + [Fact] + public void Migrated_database_allows_snapshot_isolation() + { + using var conn = _fixture.OpenConnection(); + using var cmd = new SqlCommand( + "SELECT snapshot_isolation_state_desc FROM sys.databases WHERE name = DB_NAME();", conn); + cmd.ExecuteScalar().ShouldBe("ON"); + } + + /// + /// A row committed by another connection while the snapshot read is in progress is NOT observed by + /// the tables read after it — every table reflects the instant the transaction started. Positive + /// control: a read started after the commit does see the row. + /// + [Fact] + public async Task FromConfigDb_reads_every_table_as_of_one_instant() + { + var clusterId = $"SNAP-{Guid.NewGuid():N}"[..24]; + var scriptId = $"late-{Guid.NewGuid():N}"; + await using (var seed = NewContext()) + { + seed.ServerClusters.Add(new ServerCluster + { + ClusterId = clusterId, Name = "snapshot-read", Enterprise = "zb", Site = "test", + NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + await seed.SaveChangesAsync(); + } + + try + { + var options = new DbContextOptionsBuilder() + .UseSqlServer(_fixture.ConnectionString) + .AddInterceptors(new LateEditInterceptor(_fixture.ConnectionString, clusterId, scriptId)) + .Options; + + await using (var db = new OtOpcUaConfigDbContext(options)) + { + var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(db); + // The interceptor committed a Script while ServerClusters (the FIRST table read) was being + // materialised. Scripts is read later in the same transaction and must not see it. + snapshot.ServerClusters.ShouldContain(c => c.ClusterId == clusterId); + snapshot.Scripts.ShouldNotContain(s => s.ScriptId == scriptId); + } + + await using (var verify = NewContext()) + { + // Positive control: the edit really landed, and a read started afterwards sees it. + (await verify.Scripts.AnyAsync(s => s.ScriptId == scriptId)).ShouldBeTrue(); + var later = await DraftSnapshotFactory.FromConfigDbAsync(verify); + later.Scripts.ShouldContain(s => s.ScriptId == scriptId); + } + } + finally + { + await using var cleanup = NewContext(); + await cleanup.Scripts.Where(s => s.ScriptId == scriptId).ExecuteDeleteAsync(); + await cleanup.ServerClusters.Where(c => c.ClusterId == clusterId).ExecuteDeleteAsync(); + } + } + + private OtOpcUaConfigDbContext NewContext() => new( + new DbContextOptionsBuilder().UseSqlServer(_fixture.ConnectionString).Options); + + /// + /// Commits a Script on a SEPARATE connection the first time the snapshot read materialises the + /// seeded ServerCluster row — i.e. mid-read, after the first table and before Scripts. + /// + private sealed class LateEditInterceptor(string connectionString, string clusterId, string scriptId) + : IMaterializationInterceptor + { + private int _fired; + + public object InitializedInstance(MaterializationInterceptionData materializationData, object entity) + { + if (entity is ServerCluster c && c.ClusterId == clusterId && Interlocked.Exchange(ref _fired, 1) == 0) + { + using var other = new OtOpcUaConfigDbContext( + new DbContextOptionsBuilder().UseSqlServer(connectionString).Options); + other.Scripts.Add(new Script + { + ScriptId = scriptId, Name = "late edit", SourceCode = "return 1;", SourceHash = "late-hash", + }); + other.SaveChanges(); + } + return entity; + } + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs index 9ad5806a..df1fad5a 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs @@ -103,6 +103,36 @@ public sealed class DraftSnapshotFactoryTests : IDisposable snapshot.ActiveReservations[0].ReleasedAt.ShouldBeNull(); } + /// + /// R03:3.2 — the deploy path validates AND composes from ONE snapshot, so the tables the composer + /// needs beyond the validator's set (clusters, nodes, API keys) must ride the same read. + /// + [Fact] + public async Task FromConfigDb_carries_clusters_nodes_and_api_keys_for_the_composer() + { + _db.ServerClusters.Add(new ServerCluster + { + ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", + NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + _db.ClusterNodes.Add(new ClusterNode + { + NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1", + ApplicationUri = "urn:OtOpcUa:site-a-1", Enabled = true, CreatedBy = "test", + }); + _db.ApiKeys.Add(new ApiKey + { + ApiKeyId = "k1", Name = "scada", SecretHash = "hash", Principal = "scadabridge", Enabled = true, + }); + await _db.SaveChangesAsync(); + + var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(_db); + + snapshot.ServerClusters.ShouldHaveSingleItem().ClusterId.ShouldBe("SITE-A"); + snapshot.ClusterNodes.ShouldHaveSingleItem().NodeId.ShouldBe("site-a-1:4053"); + snapshot.ApiKeys.ShouldHaveSingleItem().ApiKeyId.ShouldBe("k1"); + } + private void SeedEquipment(string equipmentId) { var uuid = Guid.NewGuid(); diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs index 0b20f490..b68fb1cd 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs @@ -55,7 +55,10 @@ public sealed class DraftValidatorCalculationTests Tags = [CalcTagRow("t-1", "dev-1", "a", "{\"changeTriggered\":true}")], }; - DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationScriptMissing" && e.Context == "t-1"); + var errors = DraftValidator.Validate(draft); + errors.ShouldContain(e => e.Code == "CalculationScriptMissing" && e.Context == "t-1"); + errors.ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger", + "an unresolved script has no reads to count — the missing script is the one error to report"); } [Fact] @@ -70,7 +73,10 @@ public sealed class DraftValidatorCalculationTests Scripts = [], }; - DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationScriptNotFound" && e.Context == "t-1"); + var errors = DraftValidator.Validate(draft); + errors.ShouldContain(e => e.Code == "CalculationScriptNotFound" && e.Context == "t-1"); + errors.ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger", + "an unresolved script has no reads to count — the missing script is the one error to report"); } [Fact] @@ -82,7 +88,9 @@ public sealed class DraftValidatorCalculationTests DriverInstances = [CalcDriver()], Devices = [Device("dev-1", "di-calc", "engine")], Tags = [CalcTagRow("t-1", "dev-1", "a", Cfg("s1"))], - Scripts = [ScriptRow("s1", "return 42.0;")], + // A read, so the change trigger is effective (a constant change-triggered script with no timer + // is now itself a deploy error — CalculationNoEffectiveTrigger). + Scripts = [ScriptRow("s1", "return (double)ctx.GetTag(\"mb/plc/reg\").Value + 42.0;")], }; DraftValidator.Validate(draft).ShouldNotContain(e => e.Code.StartsWith("Calculation")); @@ -132,6 +140,58 @@ public sealed class DraftValidatorCalculationTests msg.ShouldContain("calc/engine/b"); } + // ── effective-trigger gate (R07:U5) ───────────────────────────────────────────────────── + + [Fact] + public void Change_triggered_tag_with_no_literal_read_and_no_timer_is_rejected() + { + var draft = new DraftSnapshot + { + GenerationId = 1, ClusterId = "c", + DriverInstances = [CalcDriver()], + Devices = [Device("dev-1", "di-calc", "engine")], + Tags = [CalcTagRow("t-1", "dev-1", "a", Cfg("s1"))], + Scripts = [ScriptRow("s1", "return ctx.Now.Hour;")], + }; + + DraftValidator.Validate(draft).ShouldContain(e => + e.Code == "CalculationNoEffectiveTrigger" && e.Context == "t-1" && e.Message.Contains("no literal ctx.GetTag")); + } + + [Fact] + public void Tag_with_change_trigger_off_and_no_timer_is_rejected() + { + var draft = new DraftSnapshot + { + GenerationId = 1, ClusterId = "c", + DriverInstances = [CalcDriver()], + Devices = [Device("dev-1", "di-calc", "engine")], + Tags = [CalcTagRow("t-1", "dev-1", "a", "{\"scriptId\":\"s1\",\"changeTriggered\":false}")], + Scripts = [ScriptRow("s1", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")], + }; + + DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationNoEffectiveTrigger" && e.Context == "t-1"); + } + + [Theory] + [InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":true,\"timerIntervalMs\":1000}", "return ctx.Now.Hour;")] // timer rescues a read-less script + [InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":false,\"timerIntervalMs\":500}", "return 1.0;")] // timer-only tag + [InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":true}", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")] // change trigger with a read + [InlineData("{\"scriptId\":\"s1\"}", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")] // changeTriggered defaults to true + public void A_tag_with_an_effective_trigger_is_accepted(string cfg, string source) + { + var draft = new DraftSnapshot + { + GenerationId = 1, ClusterId = "c", + DriverInstances = [CalcDriver()], + Devices = [Device("dev-1", "di-calc", "engine")], + Tags = [CalcTagRow("t-1", "dev-1", "a", cfg)], + Scripts = [ScriptRow("s1", source)], + }; + + DraftValidator.Validate(draft).ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger"); + } + [Fact] public void Cross_driver_ref_is_terminal_not_an_edge() { diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs index 9c4c6b11..97d68679 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs @@ -342,6 +342,40 @@ public sealed class LocalDbStoreAndForwardSinkTests : IDisposable sink.GetStatus().DeadLetterDepth.ShouldBe(0, "purged past retention"); } + /// + /// Review R05:7 — GetStatus no longer runs a COUNT(*) table scan per call for the + /// dead-letter depth: it reads a cached counter kept in step by every dead-letter / revive / purge + /// path (the other tests in this class pin those), and resynced from storage on the same cadence as + /// the queued-row counter. A row flipped underneath the sink (the peer replicating a dead-letter in) + /// is therefore invisible until the next resync — that is the documented trade, and the resync is + /// what makes it bounded rather than permanent. + /// + [Fact] + public async Task DeadLetterDepth_is_served_from_a_cached_counter_that_the_periodic_resync_corrects() + { + var writer = new FakeWriter(); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); + await sink.EnqueueAsync(Event("A1"), CancellationToken.None); + await sink.EnqueueAsync(Event("A2"), CancellationToken.None); + sink.GetStatus().DeadLetterDepth.ShouldBe(0); + var probesBefore = sink.DebugCapacityProbeCount; + + // Drift the table underneath the sink, exactly as a replicated dead-letter from the pair peer would. + await Db.ExecuteAsync( + "UPDATE alarm_sf_events SET dead_lettered = 1 WHERE id IN (SELECT id FROM alarm_sf_events LIMIT 1)", + null, CancellationToken.None); + + // Cached: GetStatus did not go to the table, so it still reports the pre-drift depth… + sink.GetStatus().DeadLetterDepth.ShouldBe(0, "GetStatus serves the cached counter, not a live COUNT(*)"); + sink.DebugCapacityProbeCount.ShouldBe(probesBefore, "GetStatus must not probe storage"); + + // …and the resync (the one the enqueue cadence drives) brings both counters back in line. + await sink.DebugResyncCountersAsync(CancellationToken.None); + var status = sink.GetStatus(); + status.DeadLetterDepth.ShouldBe(1); + status.QueueDepth.ShouldBe(1); + } + /// Verifies that RetryDeadLettered requeues dead-lettered rows for retry. [Fact] public async Task RetryDeadLettered_requeues_for_retry() diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs new file mode 100644 index 00000000..2e9b7df6 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs @@ -0,0 +1,162 @@ +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; + +/// +/// Review R05:1 / R05:12 / R05:3 — the engine's value cache is pruned on every reload (it used to keep +/// every path ever referenced, forever), is no longer written after Dispose, and Dispose's in-flight +/// drain is bounded (a hung store no longer hangs host shutdown). +/// +[Trait("Category", "Unit")] +public sealed class ScriptedAlarmEngineLifecycleTests +{ + private static ScriptedAlarmDefinition Alarm(string id, string predicate) => + new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, + Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); + + private static ScriptedAlarmEngine Build(FakeUpstream up, IAlarmStateStore? store = null) + { + var logger = new LoggerConfiguration().CreateLogger(); + return new ScriptedAlarmEngine(up, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger); + } + + [Fact] + public async Task Reload_prunes_value_cache_entries_for_paths_the_new_alarm_set_no_longer_references() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + up.Set("Pressure", 5); + using var eng = Build(up); + var ct = TestContext.Current.CancellationToken; + + await eng.LoadAsync( + [Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100 || (int)ctx.GetTag("Pressure").Value > 10;""")], ct); + eng.ValueCacheCountForTest.ShouldBe(2); + + // The redeploy drops Pressure from every predicate: its entry must go with it. + await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], ct); + eng.ValueCacheCountForTest.ShouldBe(1); + eng.ValueCacheContainsForTest("Pressure").ShouldBeFalse(); + eng.ValueCacheContainsForTest("Temp").ShouldBeTrue(); + } + + [Fact] + public async Task A_push_after_Dispose_does_not_write_the_value_cache() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + var eng = Build(up); + await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + eng.Dispose(); + eng.ValueCacheCountForTest.ShouldBe(1); + + eng.OnUpstreamChange("Late", new DataValueSnapshot(1, 0u, DateTime.UtcNow, DateTime.UtcNow)); + + eng.ValueCacheContainsForTest("Late").ShouldBeFalse("a disposed engine must not grow its cache"); + } + + [Fact] + public async Task Dispose_gives_up_on_a_store_that_never_returns_after_the_drain_bound_and_cancels_it() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + var store = new NeverReturningSaveStore(); + var eng = Build(up, store); + eng.DisposeDrainTimeout = TimeSpan.FromMilliseconds(300); + await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + + store.BlockNext = true; + up.Push("Temp", 150); // Activated → SaveAsync → blocks forever (until cancelled) + await WaitForAsync(() => store.SaveInProgress); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + eng.Dispose(); + sw.Stop(); + + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the drain is bounded; a hung store cannot hang shutdown"); + await WaitForAsync(() => store.ObservedCancellation, timeoutMs: 2000); + store.ObservedCancellation.ShouldBeTrue("the in-flight save was cancelled through the engine's shutdown token"); + eng.ShutdownCtsDisposedForTest.ShouldBeTrue("the cancelled save completed inside the grace, so the CTS is released"); + } + + [Fact] + public async Task Dispose_still_waits_for_a_save_that_completes_inside_the_bound() + { + // Positive control for the bound: a save that finishes promptly is drained, not cut off. + var up = new FakeUpstream(); + up.Set("Temp", 50); + var store = new NeverReturningSaveStore { ReleaseAfter = TimeSpan.FromMilliseconds(100) }; + var eng = Build(up, store); + eng.DisposeDrainTimeout = TimeSpan.FromSeconds(5); + await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + + store.BlockNext = true; + up.Push("Temp", 150); + await WaitForAsync(() => store.SaveInProgress); + + eng.Dispose(); + + store.SaveInProgress.ShouldBeFalse(); + store.ObservedCancellation.ShouldBeFalse("a save that completes within the bound is never cancelled"); + store.Saved.ShouldContain("A"); + eng.ShutdownCtsDisposedForTest.ShouldBeTrue("every background task completed, so the shutdown CTS is disposed"); + Should.NotThrow(() => eng.OnUpstreamChange("Temp", new DataValueSnapshot(1, 0u, DateTime.UtcNow, DateTime.UtcNow))); + } + + private static async Task WaitForAsync(Func cond, int timeoutMs = 5000) + { + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs); + while (DateTime.UtcNow < deadline) + { + if (cond()) return; + await Task.Delay(20); + } + throw new TimeoutException("Condition did not become true in time"); + } + + /// A store whose next SaveAsync blocks until cancelled (or until ). + private sealed class NeverReturningSaveStore : IAlarmStateStore + { + private readonly InMemoryAlarmStateStore _inner = new(); + public volatile bool BlockNext; + public volatile bool SaveInProgress; + public volatile bool ObservedCancellation; + public TimeSpan? ReleaseAfter { get; set; } + public List Saved { get; } = []; + + public Task LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct); + public Task> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct); + public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct); + + public async Task SaveAsync(AlarmConditionState state, CancellationToken ct) + { + if (BlockNext) + { + BlockNext = false; + SaveInProgress = true; + try + { + await Task.Delay(ReleaseAfter ?? Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + ObservedCancellation = true; + throw; + } + finally + { + SaveInProgress = false; + } + } + lock (Saved) Saved.Add(state.AlarmId); + await _inner.SaveAsync(state, ct).ConfigureAwait(false); + } + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs new file mode 100644 index 00000000..ab2fde04 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs @@ -0,0 +1,150 @@ +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; + +/// +/// runs every predicate through a keyed by +/// alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation +/// spawned another abandoned run); now the orphan is one, later pushes are refused (and a +/// predicate that times out repeatedly is quarantined) — frozen at its prior state and annotated Bad quality so the operator can see it — while a +/// sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine. +/// +[Trait("Category", "Unit")] +public sealed class ScriptedAlarmEngineRunGuardTests +{ + private const uint StatusBadInternalError = 0x80020000u; + + private const string SpinPredicate = + "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return false;"; + + private static ScriptedAlarmDefinition Alarm(string id, string predicate) => + new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, + Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); + + private static ScriptedAlarmEngine Build(FakeUpstream up, ScriptingOptions? scripting = null) + { + var logger = new LoggerConfiguration().CreateLogger(); + // 500 ms, not the 60 ms the guard's own tests use: the HEALTHY alarm's first run is JIT-cold and + // must finish inside the budget on a loaded CI box, or the positive control itself times out. The + // spin predicate outlives this by 12x, so the runaway alarm still cannot finish first. + return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger, + scriptTimeout: TimeSpan.FromMilliseconds(500), scripting: scripting); + } + + [Fact] + public async Task Runaway_predicate_leaks_one_orphan_is_refused_annotated_Bad_once_and_a_sibling_keeps_evaluating() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + using var eng = Build(up); + var events = new List(); + eng.OnEvent += (_, e) => { lock (events) events.Add(e); }; + + // The load itself evaluates once (startup recovery) — that is the first, abandoned run. + await eng.LoadAsync( + [Alarm("runaway", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"), + Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + + // N more pushes while the orphan runs: every one is refused (a refusal is not a strike — I1), and + // the Bad annotation is emitted ONCE for the whole streak, not once per push (I3). + const int pushes = 4; + for (var i = 0; i < pushes; i++) + { + up.Push("Temp", 60 + i); + var expected = i + 1; + await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= expected); + } + await WaitForAsync(() => eng.RunGuard.StartedRunCount >= 1 + 1 + pushes); + + eng.RunGuard.IsQuarantined("runaway", out _).ShouldBeFalse("refusals are not strikes"); + eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one dispatch for the runaway alarm, one per evaluation for the healthy one"); + eng.RunGuard.InFlightCount.ShouldBe(1); + + // The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving — + // exactly once across the whole run of non-completions. + ScriptedAlarmEvent[] snapshot; + lock (events) snapshot = events.ToArray(); + var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray(); + quality.Length.ShouldBe(1); + quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError); + eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented"); + + // Positive control: the healthy sibling still transitions on the same engine. + up.Push("Temp", 150); + await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active); + } + + /// I3 — a QUARANTINED predicate is re-attempted on every push and refused before dispatch; the + /// Bad annotation must still be emitted once, not once per push. + [Fact] + public async Task A_quarantined_predicate_emits_QualityChanged_once_across_many_pushes() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + var events = new List(); + eng.OnEvent += (_, e) => { lock (events) events.Add(e); }; + + await eng.LoadAsync( + [Alarm("q", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"), + Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 1000;""")], + TestContext.Current.CancellationToken); + eng.RunGuard.IsQuarantined("q", out _).ShouldBeTrue(); + + // Each push re-evaluates BOTH alarms (same input). The healthy sibling's dispatch count is the + // positive control proving every push was actually processed — so the quarantined alarm was + // re-attempted N times, not coalesced into one. + const int pushes = 5; + for (var i = 0; i < pushes; i++) + { + var before = eng.RunGuard.StartedRunCount; + up.Push("Temp", 60 + i); + await WaitForAsync(() => eng.RunGuard.StartedRunCount > before); + } + eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one runaway dispatch, then healthy-only dispatches"); + await Task.Delay(300); // absence window for trailing emissions + + ScriptedAlarmEvent[] snapshot; + lock (events) snapshot = events.ToArray(); + var quality = snapshot.Where(e => e.AlarmId == "q" && e.Emission == EmissionKind.QualityChanged).ToArray(); + quality.Length.ShouldBe(1, "one Bad annotation for the quarantine, not one per push"); + quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError); + snapshot.ShouldNotContain(e => e.AlarmId == "healthy" && e.Emission == EmissionKind.QualityChanged); + } + + [Fact] + public async Task A_reload_lifts_the_quarantine() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + + await eng.LoadAsync([Alarm("a", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}")], + TestContext.Current.CancellationToken); + eng.RunGuard.IsQuarantined("a", out _).ShouldBeTrue(); + + // The operator fixed the predicate and redeployed. + await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + eng.RunGuard.IsQuarantined("a", out _).ShouldBeFalse(); + + up.Push("Temp", 150); + await WaitForAsync(() => eng.GetState("a")!.Active == AlarmActiveState.Active); + } + + private static async Task WaitForAsync(Func cond, int timeoutMs = 10_000) + { + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs); + while (DateTime.UtcNow < deadline) + { + if (cond()) return; + await Task.Delay(25); + } + throw new TimeoutException("Condition did not become true in time"); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs new file mode 100644 index 00000000..94c0408c --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs @@ -0,0 +1,59 @@ +using Serilog; +using Serilog.Core; +using Serilog.Events; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; + +/// Review R05:6 — a throwing subscriber used to be +/// swallowed with no diagnostic: a lost alarm event with no trace. It is now logged at Warning. +[Trait("Category", "Unit")] +public sealed class ScriptedAlarmSourceLoggingTests +{ + private sealed class CapturingSink : ILogEventSink + { + public List Events { get; } = []; + public void Emit(LogEvent logEvent) { lock (Events) Events.Add(logEvent); } + } + + [Fact] + public async Task A_throwing_subscriber_is_logged_at_Warning_and_does_not_crash_the_adapter() + { + var sinkCapture = new CapturingSink(); + var logger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sinkCapture).CreateLogger(); + var up = new FakeUpstream(); + up.Set("Temp", 50); + var engine = new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger); + await engine.LoadAsync( + [new ScriptedAlarmDefinition("Plant/Line1::HighTemp", "Plant/Line1", "HighTemp", AlarmKind.LimitAlarm, + AlarmSeverity.High, "Temp {Temp}C", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + using var _e = engine; + using var source = new ScriptedAlarmSource(engine, logger); + await source.SubscribeAlarmsAsync(Array.Empty(), TestContext.Current.CancellationToken); + source.OnAlarmEvent += (_, _) => throw new InvalidOperationException("subscriber broke"); + + up.Push("Temp", 150); + + var deadline = DateTime.UtcNow.AddSeconds(10); + while (DateTime.UtcNow < deadline) + { + lock (sinkCapture.Events) + { + if (sinkCapture.Events.Any(e => e.Level == LogEventLevel.Warning && e.Exception?.Message == "subscriber broke")) + break; + } + await Task.Delay(25); + } + + LogEvent warning; + lock (sinkCapture.Events) + warning = sinkCapture.Events.Single(e => e.Level == LogEventLevel.Warning && e.Exception?.Message == "subscriber broke"); + warning.RenderMessage().ShouldContain("Plant/Line1::HighTemp"); + // The adapter survived: the engine's state still moved. + engine.GetState("Plant/Line1::HighTemp")!.Active.ShouldBe(AlarmActiveState.Active); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs new file mode 100644 index 00000000..70674942 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs @@ -0,0 +1,141 @@ +using Serilog; +using Serilog.Core; +using Serilog.Events; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; + +/// +/// Per-source rate limit on the script-log fan-out (review R05:9 / R02:P6): a script that logs — or +/// fails — on every evaluation of a 1 Hz input used to put one entry per second per script onto the +/// cluster bus and the main log. The limiter admits the first 20 entries per source per minute, drops +/// the rest, and surfaces one summary line (with the dropped count) when the next entry arrives after +/// the window rolled — so nothing is silently lost, and a chatty script cannot shed everyone else's +/// entries out of the bounded telemetry channels. +/// +[Trait("Category", "Unit")] +public sealed class ScriptLogRateLimiterTests +{ + private sealed class Clock + { + public DateTime Now = new(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc); + public void Advance(TimeSpan by) => Now += by; + } + + [Fact] + public void Admits_the_first_N_per_window_then_suppresses_and_summarises_once_the_window_rolls() + { + var clock = new Clock(); + var limiter = new ScriptLogRateLimiter(limitPerWindow: 3, window: TimeSpan.FromMinutes(1), clock: () => clock.Now); + + limiter.Admit("s1").Admitted.ShouldBeTrue(); + limiter.Admit("s1").Admitted.ShouldBeTrue(); + limiter.Admit("s1").Admitted.ShouldBeTrue(); + limiter.Admit("s1").Admitted.ShouldBeFalse(); + limiter.Admit("s1").Admitted.ShouldBeFalse(); + limiter.Admit("s1").SuppressedInPreviousWindow.ShouldBe(0, "no summary until the window rolls"); + + clock.Advance(TimeSpan.FromMinutes(1)); + var first = limiter.Admit("s1"); + first.Admitted.ShouldBeTrue("a fresh window admits again"); + first.SuppressedInPreviousWindow.ShouldBe(3, "the caller emits one summary for the three dropped entries"); + limiter.Admit("s1").SuppressedInPreviousWindow.ShouldBe(0, "the summary is reported exactly once"); + } + + [Fact] + public void Sources_are_independent() + { + var limiter = new ScriptLogRateLimiter(limitPerWindow: 1); + limiter.Admit("a").Admitted.ShouldBeTrue(); + limiter.Admit("a").Admitted.ShouldBeFalse(); + limiter.Admit("b").Admitted.ShouldBeTrue("a chatty source must not shed another source's entries"); + } + + [Fact] + public void Defaults_are_20_per_minute() + { + ScriptLogRateLimiter.DefaultLimitPerWindow.ShouldBe(20); + ScriptLogRateLimiter.DefaultWindow.ShouldBe(TimeSpan.FromMinutes(1)); + var limiter = new ScriptLogRateLimiter(); + limiter.LimitPerWindow.ShouldBe(20); + new ScriptLogRateLimiter(limitPerWindow: 3).LimitPerWindow.ShouldBe(3); + new ScriptLogRateLimiter(limitPerWindow: 0).LimitPerWindow.ShouldBe(1, "values below 1 are treated as 1"); + for (var i = 0; i < 20; i++) limiter.Admit("s").Admitted.ShouldBeTrue(); + limiter.Admit("s").Admitted.ShouldBeFalse(); + } + + // ── the two sinks ──────────────────────────────────────────────────────────────────────── + + private sealed class FakePublisher : IScriptLogPublisher + { + public List Published { get; } = []; + public void Publish(ScriptLogEntry entry) => Published.Add(entry); + } + + private static LogEvent Evt(LogEventLevel level, string template, params (string Key, string Val)[] props) + { + var tmpl = new Serilog.Parsing.MessageTemplateParser().Parse(template); + var logProps = props.Select(p => new LogEventProperty(p.Key, new ScalarValue(p.Val))); + return new LogEvent(DateTimeOffset.UtcNow, level, exception: null, tmpl, logProps); + } + + [Fact] + public void Topic_sink_rate_limits_per_source_and_publishes_a_summary_entry() + { + var clock = new Clock(); + var publisher = new FakePublisher(); + var sink = new ScriptLogTopicSink(publisher, LogEventLevel.Information, + new ScriptLogRateLimiter(limitPerWindow: 2, clock: () => clock.Now)); + + for (var i = 0; i < 5; i++) + sink.Emit(Evt(LogEventLevel.Warning, "chatty", (ScriptLoggerFactory.ScriptIdProperty, "S1"), (ScriptLoggerFactory.VirtualTagIdProperty, "V1"))); + sink.Emit(Evt(LogEventLevel.Warning, "other", (ScriptLoggerFactory.ScriptIdProperty, "S2"))); + + publisher.Published.Count(e => e.ScriptId == "S1").ShouldBe(2, "the limit is per source"); + publisher.Published.Count(e => e.ScriptId == "S2").ShouldBe(1, "another source is unaffected"); + + clock.Advance(TimeSpan.FromMinutes(1)); + sink.Emit(Evt(LogEventLevel.Warning, "chatty again", (ScriptLoggerFactory.ScriptIdProperty, "S1"), (ScriptLoggerFactory.VirtualTagIdProperty, "V1"))); + + var s1 = publisher.Published.Where(e => e.ScriptId == "S1").ToList(); + s1.Count.ShouldBe(4, "2 admitted + 1 summary + 1 admitted in the new window"); + var summary = s1[2]; + summary.Message.ShouldContain("3"); + summary.Message.ShouldContain("suppressed"); + summary.Message.ShouldContain("limit 2/window", Case.Sensitive, "the summary names the CONFIGURED limit, not the default"); + summary.VirtualTagId.ShouldBe("V1", "the summary carries the source's identity so the page attributes it"); + summary.Level.ShouldBe("Warning"); + s1[3].Message.ShouldBe("chatty again"); + } + + private sealed class CapturingSink : ILogEventSink + { + public List Events { get; } = []; + public void Emit(LogEvent logEvent) => Events.Add(logEvent); + } + + [Fact] + public void Companion_sink_rate_limits_per_script_and_mirrors_a_summary_line() + { + var clock = new Clock(); + var main = new CapturingSink(); + var mainLogger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(main).CreateLogger(); + var sink = new ScriptLogCompanionSink(mainLogger, LogEventLevel.Error, + new ScriptLogRateLimiter(limitPerWindow: 2, clock: () => clock.Now)); + + for (var i = 0; i < 5; i++) + sink.Emit(Evt(LogEventLevel.Error, "boom", (ScriptLoggerFactory.ScriptNameProperty, "HighTemp"))); + main.Events.Count.ShouldBe(2); + + clock.Advance(TimeSpan.FromMinutes(1)); + sink.Emit(Evt(LogEventLevel.Error, "boom", (ScriptLoggerFactory.ScriptNameProperty, "HighTemp"))); + + main.Events.Count.ShouldBe(4); + var summary = main.Events[2].RenderMessage(); + summary.ShouldContain("HighTemp"); + summary.ShouldContain("3"); + summary.ShouldContain("suppressed"); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs new file mode 100644 index 00000000..b4efe8af --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs @@ -0,0 +1,409 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; + +/// +/// — the per-script run quarantine shared by the virtual-tag evaluator, the +/// Calculation evaluator and the scripted-alarm engine. A timed-out script is abandoned, not stopped, so +/// before the guard every re-trigger of a runaway script leaked one more thread-pool thread; the guard +/// refuses to start a run while the previous one is still executing, quarantines a script after three +/// consecutive timeouts (a refusal is not a strike), and quarantines a script whose single run allocates past the budget. +/// +[Trait("Category", "Unit")] +public sealed class ScriptRunGuardTests +{ + /// + /// A CPU-bound script that outlives every timeout in this file but still ENDS (so the orphaned pool + /// thread it leaves behind frees itself after the test). System.DateTime is outside the sandbox + /// deny-list. + /// + private const string SpinScript = + """ + var end = System.DateTime.UtcNow.AddSeconds(6); + while (System.DateTime.UtcNow < end) { } + return 1; + """; + + private const string FastScript = "return 42;"; + + /// Outlives the 60 ms budget but finishes within ~250 ms, so the slot frees up between attempts. + private const string BriefSpinScript = + """ + var end = System.DateTime.UtcNow.AddMilliseconds(250); + while (System.DateTime.UtcNow < end) { } + return 1; + """; + + private static readonly TimeSpan ShortTimeout = TimeSpan.FromMilliseconds(60); + + private static ScriptEvaluator Compile(string source) => + ScriptEvaluator.Compile(source); + + private static Task Run(ScriptRunGuard guard, string key, string source, + ScriptEvaluator evaluator, TimeSpan? timeout = null, CancellationToken ct = default) => + guard.RunAsync(key, source, evaluator, new FakeScriptContext(), timeout ?? TimeSpan.FromSeconds(5), ct); + + [Fact] + public async Task Completed_run_returns_the_value_and_releases_the_in_flight_slot() + { + var guard = new ScriptRunGuard(); + using var fast = Compile(FastScript); + + var value = await Run(guard, "vt-1", FastScript, fast, ct: TestContext.Current.CancellationToken); + + value.ShouldBe(42); + guard.InFlightCount.ShouldBe(0); + guard.StartedRunCount.ShouldBe(1); + guard.IsQuarantined("vt-1", out _).ShouldBeFalse(); + } + + [Fact] + public async Task A_timed_out_run_throws_ScriptTimeoutException_and_keeps_the_slot_held_by_the_orphan() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + + await Should.ThrowAsync(() => + Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + // The abandoned script is still executing on its pool thread — the guard knows, the caller does not. + guard.InFlightCount.ShouldBe(1); + guard.StartedRunCount.ShouldBe(1); + } + + [Fact] + public async Task A_run_while_the_previous_orphan_is_still_executing_is_refused_without_starting_another() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + await Should.ThrowAsync(() => + Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + var ex = await Should.ThrowAsync(() => + Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + ex.Message.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + guard.StartedRunCount.ShouldBe(1, "the refusal must not dispatch a second Task.Run"); + guard.InFlightCount.ShouldBe(1); + guard.RefusedWhileRunningCount.ShouldBe(1); + } + + [Fact] + public async Task Three_consecutive_timeouts_quarantine_the_script_and_a_sibling_keeps_evaluating() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(BriefSpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + // Three timeouts in a row — each orphan is allowed to finish so the next attempt is a real run, + // not a refusal (a refusal is not a strike; see the next test). + await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + await WaitForIdle(guard); + await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + await WaitForIdle(guard); + var quarantined = await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + await WaitForIdle(guard); + + quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + quarantined.Message.ShouldContain("3 consecutive"); + guard.IsQuarantined("bad", out var reason).ShouldBeTrue(); + reason.ShouldNotBeNull(); + guard.QuarantinedCount.ShouldBe(1); + + // A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run. + await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + guard.StartedRunCount.ShouldBe(3); + + // Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key. + (await Run(guard, "good", FastScript, fast, ct: ct)).ShouldBe(42); + guard.IsQuarantined("good", out _).ShouldBeFalse(); + guard.StartedRunCount.ShouldBe(4); + } + + /// + /// A refusal is NOT a strike (review fix round 1, I1): on a 1 Hz trigger with a 2 s budget a single + /// 4 s overrun is one timeout followed by two refusals, and counting those would quarantine a script + /// that merely ran long once. A refusal is a Bad publish plus its counter, and it leaves the timeout + /// streak untouched — neither adding to it nor resetting it. + /// + [Fact] + public async Task Refusals_neither_count_toward_nor_reset_the_timeout_streak() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 }); + using var spin = Compile(BriefSpinScript); + var ct = TestContext.Current.CancellationToken; + + // timeout #1, then refusals while the orphan is still running — however many, no quarantine. + await Should.ThrowAsync(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct)); + for (var i = 0; i < 5; i++) + await Should.ThrowAsync(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeFalse("refusals are not strikes"); + guard.RefusedWhileRunningCount.ShouldBe(5); + await WaitForIdle(guard); + + // …and they did not RESET the streak either: the second real timeout reaches the threshold of 2. + await Should.ThrowAsync(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeTrue(); + await WaitForIdle(guard); + } + + /// + /// The thread-leak regression, and the stuck-orphan quarantine (fix round 2). Pre-guard, a + /// while(true) script on a 1 Hz trigger leaked one pool thread per trigger — 30 orphans after 30 + /// triggers, trending toward pool starvation. Now the first trigger's orphan is the only one; later + /// triggers are refused while it is young, and once it has run longer than + /// × the timeout (10 × 60 ms here — the + /// trigger cadence is scaled down from 1 Hz to keep the test short) the next refusal quarantines it. + /// A healthy sibling keeps evaluating throughout (the positive control an absence assertion needs). + /// + /// + /// The load-bearing assertions are on the guard's own bookkeeping ( + /// / ): moves with + /// every parallel test class on a loaded machine, so it is kept as a coarse secondary check only — + /// it must stay far below the 30 the leak would have produced. + /// + [Fact] + public async Task Runaway_script_leaves_one_orphan_is_quarantined_once_stuck_and_a_sibling_keeps_evaluating() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + var threadsBefore = ThreadPool.ThreadCount; + var stuckAfter = ShortTimeout * ScriptingOptions.DefaultStuckOrphanQuarantineFactor; // 600 ms + + var healthyResults = new List(); + var outcomes = new List(); + var sw = System.Diagnostics.Stopwatch.StartNew(); + for (var trigger = 0; trigger < 30; trigger++) + { + try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); outcomes.Add("value"); } + catch (ScriptTimeoutException) { outcomes.Add("timeout"); } + catch (ScriptRunRefusedException) { outcomes.Add(sw.Elapsed < stuckAfter ? "refused-young" : "refused"); } + catch (ScriptQuarantinedException) { outcomes.Add("quarantined"); } + + healthyResults.Add(await Run(guard, "healthy", FastScript, fast, ct: ct)); + await Task.Delay(50, ct); + } + + guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing"); + guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling"); + outcomes[0].ShouldBe("timeout"); + outcomes.ShouldContain("refused-young", "a young orphan is only refused — a short overrun is not quarantined"); + guard.IsQuarantined("runaway", out var reason).ShouldBeTrue("the orphan outlived 10x its budget: it is stuck"); + reason!.ShouldContain("stuck"); + outcomes[^1].ShouldBe("quarantined"); + guard.QuarantinedCount.ShouldBe(1); + healthyResults.ShouldAllBe(v => v == 42); + healthyResults.Count.ShouldBe(30); + + var threadDelta = ThreadPool.ThreadCount - threadsBefore; + threadDelta.ShouldBeLessThan(15, "thirty leaked orphans would have grown the pool by ~30 threads"); + } + + /// + /// Positive control for the stuck rule: a 2x overrun (one timeout, refusals while the orphan is young) + /// is NOT quarantined, and once the orphan completes the script evaluates normally again. + /// + [Fact] + public async Task A_2x_overrun_is_not_quarantined_and_resumes_evaluating_once_the_orphan_completes() + { + var guard = new ScriptRunGuard(); + const string source = + """ + var ms = (int)(ctx.GetTag("spinMs").Value ?? 0); + var end = System.DateTime.UtcNow.AddMilliseconds(ms); + while (System.DateTime.UtcNow < end) { } + return 7; + """; + using var eval = Compile(source); + var ct = TestContext.Current.CancellationToken; + var timeout = TimeSpan.FromMilliseconds(100); + + FakeScriptContext Ctx(int spinMs) + { + var c = new FakeScriptContext(); + c.Tags["spinMs"] = new Core.Abstractions.DataValueSnapshot(spinMs, 0u, c.Now, c.Now); + return c; + } + + // Warm the JIT so the first measured run's overrun is the script's, not the compiler's. + (await guard.RunAsync("k", source, eval, Ctx(0), TimeSpan.FromSeconds(5), ct)).ShouldBe(7); + + // 2x overrun: 200 ms against a 100 ms budget — times out, then is refused while it finishes. + await Should.ThrowAsync(() => guard.RunAsync("k", source, eval, Ctx(200), timeout, ct)); + await Should.ThrowAsync(() => guard.RunAsync("k", source, eval, Ctx(0), timeout, ct)); + await WaitForIdle(guard); + + guard.IsQuarantined("k", out _).ShouldBeFalse("a 2x overrun is far below the 10x stuck threshold"); + (await guard.RunAsync("k", source, eval, Ctx(0), timeout, ct)).ShouldBe(7, "the script resumes evaluating"); + guard.QuarantinedCount.ShouldBe(0); + } + + /// The stuck rule is a knob: with ≤ 0 a + /// stuck orphan is only ever refused. + [Fact] + public async Task StuckOrphanQuarantineFactor_zero_disables_the_stuck_rule() + { + var guard = new ScriptRunGuard(new ScriptingOptions { StuckOrphanQuarantineFactor = 0 }); + using var spin = Compile(SpinScript); + var ct = TestContext.Current.CancellationToken; + + await Should.ThrowAsync(() => Run(guard, "k", SpinScript, spin, ShortTimeout, ct)); + await Task.Delay(ShortTimeout * 12, ct); + await Should.ThrowAsync(() => Run(guard, "k", SpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeFalse(); + } + + [Fact] + public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("vt", out _).ShouldBeTrue(); + + // The operator fixed the script: a different source identity under the same key starts clean. The + // old orphan is a sunk cost — it keeps running, but it must not block the FIXED script. + (await Run(guard, "vt", FastScript, fast, ct: ct)).ShouldBe(42); + guard.IsQuarantined("vt", out _).ShouldBeFalse(); + guard.InFlightCount.ShouldBe(1, "the abandoned orphan is still counted until it finishes"); + } + + [Fact] + public async Task Reset_and_ResetAll_clear_a_quarantine() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + await Should.ThrowAsync(() => Run(guard, "a", SpinScript, spin, ShortTimeout, ct)); + await Should.ThrowAsync(() => Run(guard, "b", SpinScript, spin, ShortTimeout, ct)); + guard.QuarantinedKeys.ShouldBe(new[] { "a", "b" }, ignoreOrder: true); + + guard.Reset("a"); + guard.IsQuarantined("a", out _).ShouldBeFalse(); + guard.IsQuarantined("b", out _).ShouldBeTrue(); + + guard.ResetAll(); + guard.IsQuarantined("b", out _).ShouldBeFalse(); + // The key is usable again (a fresh slot: the orphan from the quarantined run no longer blocks it). + (await Run(guard, "b", FastScript, fast, ct: ct)).ShouldBe(42); + } + + [Fact] + public async Task A_run_that_allocates_past_the_budget_is_quarantined() + { + var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 1024 * 1024 }); + // Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the + // allocation as dead code (which is exactly what it does to the naive one-liner). + const string hungry = + "long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;"; + using var evaluator = Compile(hungry); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + var ex = await Should.ThrowAsync(() => Run(guard, "hog", hungry, evaluator, ct: ct)); + ex.Message.ShouldContain("allocat"); + guard.IsQuarantined("hog", out var reason).ShouldBeTrue(); + reason.ShouldNotBeNull(); + reason.ShouldContain("allocat"); + + // Never dispatched again, and the slot was released (the run DID complete). + await Should.ThrowAsync(() => Run(guard, "hog", hungry, evaluator, ct: ct)); + guard.StartedRunCount.ShouldBe(1); + guard.InFlightCount.ShouldBe(0); + + // Positive control: a modest allocation under the budget is fine. + (await Run(guard, "ok", FastScript, fast, ct: ct)).ShouldBe(42); + } + + [Fact] + public async Task A_disabled_allocation_budget_never_quarantines() + { + var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 0 }); + // Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the + // allocation as dead code (which is exactly what it does to the naive one-liner). + const string hungry = + "long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;"; + using var evaluator = Compile(hungry); + + (await Run(guard, "hog", hungry, evaluator, ct: TestContext.Current.CancellationToken)).ShouldBe(32 * (256 * 1024 + 1)); + guard.IsQuarantined("hog", out _).ShouldBeFalse(); + } + + [Fact] + public async Task A_script_that_throws_propagates_unwrapped_and_counts_as_a_completion() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 }); + const string throwing = "throw new System.InvalidOperationException(\"boom\");"; + using var thrower = Compile(throwing); + // A spin that outlives the timeout but finishes quickly, so the slot frees up between attempts. + const string briefSpin = + """ + var end = System.DateTime.UtcNow.AddMilliseconds(250); + while (System.DateTime.UtcNow < end) { } + return 1; + """; + using var spin = Compile(briefSpin); + var ct = TestContext.Current.CancellationToken; + + // timeout #1 + await Should.ThrowAsync(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct)); + await WaitForIdle(guard); + + // A throw is a completion — the script returned control — so the consecutive counter resets… + var ex = await Should.ThrowAsync(() => Run(guard, "k", throwing, thrower, ct: ct)); + ex.Message.ShouldBe("boom"); + guard.InFlightCount.ShouldBe(0); + + // …which is why this second timeout does NOT reach the threshold of 2. + await Should.ThrowAsync(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeFalse(); + await WaitForIdle(guard); + } + + [Fact] + public async Task Caller_cancellation_wins_over_the_timeout() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(80)); + + await Should.ThrowAsync(() => + Run(guard, "k", SpinScript, spin, TimeSpan.FromSeconds(5), cts.Token)); + guard.IsQuarantined("k", out _).ShouldBeFalse(); + } + + [Fact] + public async Task Quarantine_is_observable_through_the_event_and_the_counter() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + using var spin = Compile(SpinScript); + var seen = new List<(string Key, string Reason)>(); + guard.Quarantined += (_, e) => seen.Add((e.Key, e.Reason)); + + await Should.ThrowAsync(() => + Run(guard, "k", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + seen.ShouldHaveSingleItem(); + seen[0].Key.ShouldBe("k"); + seen[0].Reason.ShouldContain("consecutive"); + guard.QuarantinedCount.ShouldBe(1); + } + + private static async Task WaitForIdle(ScriptRunGuard guard) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (guard.InFlightCount != 0 && DateTime.UtcNow < deadline) + await Task.Delay(10); + guard.InFlightCount.ShouldBe(0, "the brief spin must have finished"); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs new file mode 100644 index 00000000..5c932832 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs @@ -0,0 +1,148 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; + +/// +/// Sandbox hardening (review R05:10): System.Threading is denied as a namespace with a +/// type-granular allow-list (CancellationToken, Interlocked, Volatile) so a +/// script can no longer park its pool thread on a blocking primitive or touch a system-global kernel +/// object; System.Type's reflection entry points (GetType(string), GetMethod*, +/// InvokeMember, …) and System.Runtime.CompilerServices.RuntimeHelpers are denied so a +/// forbidden type cannot be reached by name without ever being named syntactically. +/// +[Trait("Category", "Unit")] +public sealed class ScriptSandboxHardeningTests +{ + private static ScriptSandboxViolationException Rejects(string source) => + Should.Throw(() => + ScriptEvaluator.Compile(source)); + + // ── System.Threading blocking / synchronisation primitives ─────────────────────────────── + + [Fact] + public void Rejects_SemaphoreSlim_Wait() + { + var ex = Rejects("new System.Threading.SemaphoreSlim(0).Wait(); return 0;"); + ex.Rejections.ShouldContain(r => r.TypeName.Contains("SemaphoreSlim") && r.Namespace == "System.Threading"); + } + + [Fact] + public void Rejects_SpinWait_SpinUntil() + => Rejects("System.Threading.SpinWait.SpinUntil(() => false); return 0;"); + + [Fact] + public void Rejects_Monitor() + => Rejects("var o = new object(); System.Threading.Monitor.Enter(o); return 0;"); + + [Fact] + public void Rejects_named_Mutex_system_global_kernel_object() + => Rejects("using var m = new System.Threading.Mutex(false, \"Global\\\\otopcua\"); return 0;"); + + [Fact] + public void Rejects_ManualResetEventSlim() + => Rejects("var e = new System.Threading.ManualResetEventSlim(false); e.Wait(); return 0;"); + + [Fact] + public void Rejects_EventWaitHandle() + => Rejects("var e = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset); return 0;"); + + [Fact] + public void Rejects_Thread_ThreadPool_and_Timer_still() + { + Rejects("var t = new System.Threading.Thread(() => { }); return 0;"); + Rejects("System.Threading.ThreadPool.QueueUserWorkItem(_ => { }); return 0;"); + Rejects("var t = new System.Threading.Timer(_ => { }); return 0;"); + } + + [Fact] + public void Rejects_Thread_Sleep() + => Rejects("System.Threading.Thread.Sleep(10); return 0;"); + + // ── the allow-list inside System.Threading ─────────────────────────────────────────────── + + [Fact] + public async Task Allows_CancellationToken() + { + using var ev = ScriptEvaluator.Compile( + "var t = System.Threading.CancellationToken.None; return t.IsCancellationRequested ? 1 : 0;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(0); + } + + [Fact] + public async Task Allows_Interlocked() + { + using var ev = ScriptEvaluator.Compile( + "var x = 41; System.Threading.Interlocked.Increment(ref x); return x;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(42); + } + + [Fact] + public async Task Allows_Volatile() + { + using var ev = ScriptEvaluator.Compile( + "var x = 42; return System.Threading.Volatile.Read(ref x);"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(42); + } + + // ── System.Type reflection entry points ────────────────────────────────────────────────── + + [Fact] + public void Rejects_Type_GetType_by_name() + { + var ex = Rejects("var t = System.Type.GetType(\"System.IO.File\"); return t is null ? 0 : 1;"); + ex.Rejections.ShouldContain(r => r.Message.Contains("GetType")); + } + + [Fact] + public void Rejects_Type_GetMethod() + => Rejects("var m = typeof(string).GetMethod(\"Trim\"); return m is null ? 0 : 1;"); + + [Fact] + public void Rejects_Type_GetMethods() + => Rejects("return typeof(string).GetMethods().Length;"); + + [Fact] + public void Rejects_Type_InvokeMember() + => Rejects("""return (int)typeof(string).InvokeMember("Length", System.Reflection.BindingFlags.GetProperty, null, "abc", null)!;"""); + + [Fact] + public void Rejects_Type_GetProperty_GetField_GetMember_GetConstructor_GetEvent() + { + Rejects("return typeof(string).GetProperty(\"Length\") is null ? 0 : 1;"); + Rejects("return typeof(string).GetField(\"Empty\") is null ? 0 : 1;"); + Rejects("return typeof(string).GetMember(\"Length\").Length;"); + Rejects("return typeof(string).GetConstructors().Length;"); + Rejects("return typeof(string).GetEvent(\"x\") is null ? 0 : 1;"); + } + + [Fact] + public async Task Allows_instance_GetType_and_Type_metadata_reads() + { + // object.GetType() is declared on System.Object, and Type.Name / IsValueType are plain metadata + // reads — neither is a reflection entry point, and both are common in authored scripts. + using var ev = ScriptEvaluator.Compile( + "object o = 42; var t = o.GetType(); return t.IsValueType && typeof(int).Name == \"Int32\" ? 1 : 0;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(1); + } + + // ── RuntimeHelpers ─────────────────────────────────────────────────────────────────────── + + [Fact] + public void Rejects_RuntimeHelpers() + { + var ex = Rejects( + "var o = System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(string)); return o is null ? 0 : 1;"); + ex.Rejections.ShouldContain(r => r.TypeName.Contains("RuntimeHelpers")); + } + + [Fact] + public async Task Allows_compile_time_only_CompilerServices_attributes() + { + // The namespace stays open for the harmless compile-time types (MethodImplOptions etc.). + using var ev = ScriptEvaluator.Compile( + "var o = System.Runtime.CompilerServices.MethodImplOptions.NoInlining; return (int)o;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(8); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs new file mode 100644 index 00000000..0372ea15 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs @@ -0,0 +1,189 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Serilog.Events; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests; + +/// +/// Review 07 Calculation remediation: a compile failure is negative-cached by source hash and its +/// Warning/script-log fires on the Good→Bad transition only (P4); inputs are recorded before +/// InitializeAsync so a Restart computes from current inputs (H6); array results dedup +/// structurally (H8); ShutdownAsync drops the compiled scripts (L2). +/// +public sealed class CalculationDriverRemediationTests +{ + private const uint StatusGood = 0u; + private const uint StatusBadInternalError = 0x80020000u; + + private static RawTagEntry CalcTag(string rawPath, string source, bool changeTriggered = true, int? timerMs = null) + { + var timer = timerMs is { } ms ? $",\"timerIntervalMs\":{ms}" : ""; + var json = $"{{\"scriptId\":\"s-{rawPath}\",\"changeTriggered\":{(changeTriggered ? "true" : "false")}{timer}," + + $"\"scriptSource\":{System.Text.Json.JsonSerializer.Serialize(source)}}}"; + return new RawTagEntry(rawPath, json, WriteIdempotent: false); + } + + private static (CalculationDriver Driver, ConcurrentQueue Published, List Logs, CapturingLogger HostLog) + Build(params RawTagEntry[] tags) + { + var logs = new List(); + var sink = new ScriptLogTopicSink(new CapturingPublisher(logs), LogEventLevel.Information); + var scriptRoot = new ScriptRootLogger(new LoggerConfiguration().WriteTo.Sink(sink).CreateLogger()); + var hostLog = new CapturingLogger(); + var driver = new CalculationDriver(new CalculationDriverOptions { RawTags = tags }, "calc-drv", scriptRoot, hostLog); + var published = new ConcurrentQueue(); + driver.OnDataChange += (_, e) => published.Enqueue(e); + return (driver, published, logs, hostLog); + } + + private static async Task Settle(CalculationDriver driver) + { + var deadline = DateTime.UtcNow.AddSeconds(30); + while (!driver.IsEvaluationIdle && DateTime.UtcNow < deadline) await Task.Delay(10); + driver.IsEvaluationIdle.ShouldBeTrue("the calc driver's evaluation pump should have drained"); + } + + /// Fix round 1 minor: Dispose releases the evaluation gate and its cancellation source — even + /// with a runaway run in flight (the guard lets the caller's cancel win, so the awaited run returns + /// promptly) — and a trigger arriving after Dispose is harmless. + [Fact] + public async Task Dispose_with_a_run_in_flight_disposes_the_eval_gate_and_a_late_trigger_is_harmless() + { + var (driver, _, _, _) = Build( + CalcTag("calc/e/spin", "var v = ctx.GetTag(\"src/a\").Value; var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 1;")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); + var deadline = DateTime.UtcNow.AddSeconds(10); + while (driver.IsEvaluationIdle && DateTime.UtcNow < deadline) await Task.Delay(5); + driver.IsEvaluationIdle.ShouldBeFalse("positive control: a run is in flight when Dispose lands"); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + driver.Dispose(); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the caller's cancel releases the awaited run; Dispose is not held for the 6 s spin"); + driver.EvalResourcesDisposedForTest.ShouldBeTrue(); + + Should.NotThrow(() => driver.OnDependencyValue("src/a", 2, StatusGood, DateTime.UtcNow)); + Should.NotThrow(() => driver.Dispose()); + } + + [Fact] + public async Task A_compile_failure_is_remembered_and_logged_once_across_repeated_triggers() + { + var (driver, published, logs, hostLog) = Build( + CalcTag("calc/e/broken", "var x = (double)ctx.GetTag(\"src/a\").Value; this is not valid C#;")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + for (var i = 0; i < 5; i++) + { + driver.OnDependencyValue("src/a", i, StatusGood, DateTime.UtcNow); + await Settle(driver); + } + + published.Count.ShouldBe(1, "Bad is published once per transition"); + published.TryDequeue(out var bad).ShouldBeTrue(); + bad!.Snapshot.StatusCode.ShouldBe(StatusBadInternalError); + logs.Count.ShouldBe(1, "the script-log entry fires on the Good→Bad transition only"); + hostLog.Entries.Count(e => e.Level == LogLevel.Warning && e.Message.Contains("compile failed")).ShouldBe(1, + "Roslyn is consulted once per source; later triggers hit the negative cache"); + driver.CompileFailureCountForTest.ShouldBe(1); + } + + [Fact] + public async Task A_changed_failure_reason_is_logged_even_while_already_Bad() + { + // Throws with a different message depending on the input, so the failure REASON changes while the + // published status stays Bad — the operator must see the new reason. + var (driver, published, logs, _) = Build( + CalcTag("calc/e/thrower", "throw new System.InvalidOperationException(\"boom \" + (int)ctx.GetTag(\"src/a\").Value);")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); + await Settle(driver); + driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); // same reason → no new log + await Settle(driver); + driver.OnDependencyValue("src/a", 2, StatusGood, DateTime.UtcNow); // new reason → logged + await Settle(driver); + + published.Count.ShouldBe(1); + logs.Count.ShouldBe(2); + logs[0].Message.ShouldContain("boom 1"); + logs[1].Message.ShouldContain("boom 2"); + } + + [Fact] + public async Task Inputs_delivered_before_InitializeAsync_are_kept_so_the_first_trigger_computes() + { + // H6: a Restart is a new driver whose adapter registers interest at construction; the mux's + // retained-value replay therefore lands BEFORE InitializeAsync. It used to be dropped. + var (driver, published, _, _) = Build( + CalcTag("calc/e/sum", "return (double)ctx.GetTag(\"src/a\").Value + (double)ctx.GetTag(\"src/b\").Value;")); + + driver.OnDependencyValue("src/a", 10.0, StatusGood, DateTime.UtcNow); // replayed, pre-init + driver.OnDependencyValue("src/b", 20.0, StatusGood, DateTime.UtcNow); + published.ShouldBeEmpty("evaluation is gated on running"); + + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + driver.OnDependencyValue("src/a", 12.0, StatusGood, DateTime.UtcNow); // only ONE input moves after init + await Settle(driver); + + published.Count.ShouldBe(1, "src/b's pre-init value satisfied the change-gate"); + published.TryDequeue(out var e).ShouldBeTrue(); + e!.Snapshot.Value.ShouldBe(32.0); + } + + [Fact] + public async Task An_equal_array_result_is_deduped_structurally() + { + var (driver, published, _, _) = Build( + CalcTag("calc/e/arr", "var v = (double)ctx.GetTag(\"src/a\").Value; return new double[] { v, v * 2 };")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); + await Settle(driver); + driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); // same array contents + await Settle(driver); + published.Count.ShouldBe(1, "a structurally equal array is not a change"); + + driver.OnDependencyValue("src/a", 2.0, StatusGood, DateTime.UtcNow); + await Settle(driver); + published.Count.ShouldBe(2, "a different array still publishes"); + published.Last().Snapshot.Value.ShouldBe(new double[] { 2.0, 4.0 }); + } + + [Fact] + public async Task ShutdownAsync_drops_the_compiled_scripts() + { + var (driver, published, _, _) = Build(CalcTag("calc/e/x", "return (double)ctx.GetTag(\"src/a\").Value * 2.0;")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); + await Settle(driver); + published.Count.ShouldBe(1); + driver.CompiledScriptCountForTest.ShouldBe(1); + + await driver.ShutdownAsync(TestContext.Current.CancellationToken); + + driver.CompiledScriptCountForTest.ShouldBe(0, "the collectible ALCs must not survive the generation"); + } + + private sealed class CapturingPublisher(List entries) : IScriptLogPublisher + { + public void Publish(ScriptLogEntry entry) { lock (entries) entries.Add(entry); } + } + + internal sealed class CapturingLogger : ILogger + { + public ConcurrentBag<(LogLevel Level, string Message)> Entries { get; } = new(); + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, + Func formatter) + => Entries.Add((logLevel, formatter(state, exception))); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs index 5eed4c6b..cf22038b 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs @@ -229,11 +229,13 @@ public sealed class CalculationDriverTests logs[0].ScriptId.ShouldBe("s-calc/e/div"); logs[0].VirtualTagId.ShouldBe("calc/e/div"); - // a = 0 again → still failing, but NOT a fresh transition → no second Bad publish (script-log still emits). + // a = 0 again → still failing for the SAME reason, NOT a fresh transition → no second Bad publish, + // and (R07:P4) no second script-log entry either: the log fires on the transition or on a change of + // reason, not on every trigger of a tag that is already Bad. driver.OnDependencyValue("src/a", 0, StatusGood, DateTime.UtcNow); await Settle(driver); published.ShouldBeEmpty(); - logs.Count.ShouldBe(2); + logs.Count.ShouldBe(1); // a = 5 → 20 → Good recovery is force-published. driver.OnDependencyValue("src/a", 5, StatusGood, DateTime.UtcNow); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs new file mode 100644 index 00000000..679e7f71 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs @@ -0,0 +1,87 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests; + +/// +/// runs every compiled script through a +/// keyed by calc-tag RawPath. A timed-out run is abandoned on its pool thread; before the guard, every +/// later trigger of the same tag stacked another orphan behind it (one leaked thread per trigger). Now +/// every later trigger is refused ("previous evaluation still running" — Bad, but not a quarantine +/// strike; only consecutive timeouts quarantine), and a healthy sibling on the same evaluator keeps computing. +/// +public sealed class CalculationEvaluatorRunGuardTests +{ + private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger()); + + /// Outlives the 60 ms budget by a wide margin but still ends, so the orphan frees itself. + private const string SpinScript = + "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;"; + + private static readonly Dictionary NoDeps = new(); + + [Fact] + public async Task Triggers_during_the_previous_orphan_are_refused_not_quarantined_and_a_sibling_keeps_computing() + { + using var sut = new CalculationEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60)); + var ct = TestContext.Current.CancellationToken; + + var first = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct); + first.Success.ShouldBeFalse(); + first.Reason.ShouldNotBeNull(); + first.Reason.ShouldContain("timed out"); + + // Every later trigger while the orphan runs is a Bad "previous evaluation still running" — a + // refusal is not a strike, so the tag is NOT quarantined by them (fix round 1, I1). + for (var i = 0; i < 3; i++) + { + var refused = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); + refused.Success.ShouldBeFalse(); + refused.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + } + sut.QuarantinedScripts.ShouldBeEmpty(); + sut.RunGuard.RefusedWhileRunningCount.ShouldBe(3); + + // Exactly one run was ever dispatched for the runaway tag — the leak is one orphan, not one per trigger. + sut.RunGuard.StartedRunCount.ShouldBe(1); + sut.RunGuard.InFlightCount.ShouldBe(1); + + // Positive control: a sibling tag on the same evaluator still computes. + var sibling = await sut.EvaluateAsync("calc/ok", "return 42;", Snap(NoDeps), ct); + sibling.Success.ShouldBeTrue(sibling.Reason); + sibling.Value.ShouldBe(42); + } + + [Fact] + public async Task ClearCompiledScripts_lifts_the_quarantine_as_a_redeploy_does() + { + using var sut = new CalculationEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60), + new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + var ct = TestContext.Current.CancellationToken; + + var r = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct); + r.Reason.ShouldNotBeNull(); + r.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + + sut.ClearCompiledScripts(); + + sut.QuarantinedScripts.ShouldBeEmpty(); + // The fixed script (new source) runs; the old orphan is a sunk cost that no longer blocks the tag. + var fixedRun = await sut.EvaluateAsync("calc/loop", "return 7;", Snap(NoDeps), ct); + fixedRun.Success.ShouldBeTrue(fixedRun.Reason); + fixedRun.Value.ShouldBe(7); + } + + private static Dictionary Snap(Dictionary deps) + { + var now = DateTime.UtcNow; + var d = new Dictionary(StringComparer.Ordinal); + foreach (var kv in deps) d[kv.Key] = new Core.Abstractions.DataValueSnapshot(kv.Value, 0u, now, now); + return d; + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyHierarchySourcePagingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyHierarchySourcePagingTests.cs new file mode 100644 index 00000000..d0767592 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyHierarchySourcePagingTests.cs @@ -0,0 +1,92 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse; + +/// +/// Review 2026-10-21 G-5: Repository.DiscoverPageSize was authored and parsed but reached +/// nothing — the client's typed DiscoverHierarchyAsync hard-codes 5000. The source now pages +/// the raw RPC itself at the configured size. +/// +public sealed class GatewayGalaxyHierarchySourcePagingTests +{ + private static GalaxyObject Obj(int id) => new() { GobjectId = id, TagName = $"Obj{id}" }; + + /// Every request carries the configured page size and the page tokens chain until the last page. + [Fact] + public async Task Pages_at_the_configured_size_and_chains_tokens() + { + var requests = new List(); + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + requests.Add(req); + var reply = new DiscoverHierarchyReply(); + switch (req.PageToken) + { + case "": + reply.Objects.Add([Obj(1), Obj(2)]); + reply.NextPageToken = "p2"; + break; + case "p2": + reply.Objects.Add([Obj(3), Obj(4)]); + reply.NextPageToken = "p3"; + break; + default: + reply.Objects.Add(Obj(5)); + reply.NextPageToken = ""; + break; + } + return Task.FromResult(reply); + } + + var objects = await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 2, CancellationToken.None); + + objects.Select(o => o.GobjectId).ShouldBe([1, 2, 3, 4, 5]); + requests.Count.ShouldBe(3); + requests.ShouldAllBe(r => r.PageSize == 2); + requests.Select(r => r.PageToken).ShouldBe(["", "p2", "p3"]); + } + + /// A different configured size is what goes on the wire (positive control for the knob). + [Fact] + public async Task A_different_page_size_is_sent_on_the_wire() + { + int? sent = null; + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + sent = req.PageSize; + return Task.FromResult(new DiscoverHierarchyReply()); + } + + await GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 250, CancellationToken.None); + sent.ShouldBe(250); + } + + /// A non-positive configured size falls back to the client library's 5000 default. + [Theory] + [InlineData(0)] + [InlineData(-7)] + public void Non_positive_page_size_falls_back_to_the_default(int configured) + => GatewayGalaxyHierarchySource.EffectivePageSize(configured).ShouldBe(GatewayGalaxyHierarchySource.DefaultPageSize); + + /// A repeated page token is refused rather than walked forever. + [Fact] + public async Task A_repeated_page_token_is_refused() + { + var calls = 0; + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + calls++; + var reply = new DiscoverHierarchyReply { NextPageToken = "same" }; + reply.Objects.Add(Obj(calls)); + return Task.FromResult(reply); + } + + var ex = await Should.ThrowAsync( + () => GatewayGalaxyHierarchySource.PageAsync(Discover, pageSize: 10, CancellationToken.None)); + ex.Message.ShouldContain("repeated page token"); + calls.ShouldBe(2); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs new file mode 100644 index 00000000..a385ecfb --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Browse/GatewayGalaxyPlatformSourceTests.cs @@ -0,0 +1,60 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Browse; + +/// The platform-only query the driver seeds its ScanState probe watcher from. +public sealed class GatewayGalaxyPlatformSourceTests +{ + private static GalaxyObject Obj(string tag, params string[] chain) + { + var o = new GalaxyObject { TagName = tag }; + o.TemplateChain.Add(chain); + return o; + } + + /// Every request carries the platform template filter, no attributes, and the page size; replies are re-filtered. + [Fact] + public async Task Requests_filter_on_platform_templates_without_attributes_and_replies_are_refiltered() + { + var requests = new List(); + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + requests.Add(req); + var reply = new DiscoverHierarchyReply(); + if (req.PageToken == "") + { + reply.Objects.Add([Obj("Plat01", "$WinPlatform"), Obj("Tank01", "$UserDefined")]); + reply.NextPageToken = "p2"; + } + else + { + reply.Objects.Add([Obj("Eng01", "$AppEngine", "$WinPlatform"), Obj("Plat01", "$WinPlatform"), Obj("", "$WinPlatform")]); + } + return Task.FromResult(reply); + } + + var names = await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 50, CancellationToken.None); + + names.ShouldBe(["Plat01", "Eng01"]); + requests.Count.ShouldBe(2); + requests.ShouldAllBe(r => r.PageSize == 50 && r.IncludeAttributes == false); + requests.ShouldAllBe(r => r.TemplateChainContains.Contains("$WinPlatform") && r.TemplateChainContains.Contains("$AppEngine")); + } + + /// A gateway that strips the template chain is trusted to have filtered server-side. + [Fact] + public async Task Objects_without_a_template_chain_are_kept() + { + Task Discover(DiscoverHierarchyRequest req, CancellationToken ct) + { + var reply = new DiscoverHierarchyReply(); + reply.Objects.Add(new GalaxyObject { TagName = "Plat01" }); + return Task.FromResult(reply); + } + + (await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 0, CancellationToken.None)).ShouldBe(["Plat01"]); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/GalaxyDriverProbeTimeoutTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/GalaxyDriverProbeTimeoutTests.cs new file mode 100644 index 00000000..1f5fbb78 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/GalaxyDriverProbeTimeoutTests.cs @@ -0,0 +1,69 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests; + +/// +/// Review 2026-10-21 G-6: ProbeTimeoutSeconds was authored and compared for equivalence but +/// never consumed — the probe ran on the caller's budget alone. It now caps the probe deadline +/// (it can shorten the caller's budget, never extend it). +/// +[Trait("Category", "Unit")] +public sealed class GalaxyDriverProbeTimeoutTests +{ + /// The effective deadline is the shorter of the caller's budget and the configured cap. + [Theory] + [InlineData(10, 30, 10)] // caller shorter → caller wins + [InlineData(10, 2, 2)] // config shorter → config caps + [InlineData(10, 0, 10)] // unset config → caller + [InlineData(10, -1, 10)] // nonsense config → caller + [InlineData(0, 5, 1)] // no caller budget → 1 s floor beats a longer config + [InlineData(0, 0, 1)] // nothing → 1 s floor + public void EffectiveTimeout_is_the_shorter_of_caller_and_config(int callerSeconds, int configSeconds, int expectedSeconds) + => GalaxyDriverProbe.EffectiveTimeout(TimeSpan.FromSeconds(callerSeconds), configSeconds) + .ShouldBe(TimeSpan.FromSeconds(expectedSeconds)); + + /// + /// Against a port that accepts TCP but never speaks gRPC, a 1 s configured cap returns long + /// before the caller's 20 s budget. The gRPC handshake itself does the waiting here, so this + /// exercises the real deadline path, not the pure helper. + /// + [Fact] + public async Task Configured_cap_shortens_the_probe_against_a_silent_listener() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + try + { + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + var config = $$""" + { + "Gateway": { "Endpoint": "http://127.0.0.1:{{port}}", "ApiKeySecretRef": "dev:k", "UseTls": false }, + "MxAccess": { "ClientName": "probe-timeout" }, + "Repository": {}, + "Reconnect": {}, + "ProbeTimeoutSeconds": 1 + } + """; + + var sw = Stopwatch.StartNew(); + var result = await new GalaxyDriverProbe().ProbeAsync(config, TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken); + sw.Stop(); + + result.Ok.ShouldBeFalse(); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), $"the 1 s configured cap must bound the probe (message: {result.Message})"); + result.Message.ShouldNotBeNull(); + (result.Message!.Contains("timed out", StringComparison.OrdinalIgnoreCase) + || result.Message.Contains("DeadlineExceeded", StringComparison.OrdinalIgnoreCase) + || result.Message.Contains("handshake failed", StringComparison.OrdinalIgnoreCase)) + .ShouldBeTrue($"unexpected message: '{result.Message}'"); + } + finally + { + listener.Stop(); + } + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Health/PerPlatformProbeWatcherRegistryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Health/PerPlatformProbeWatcherRegistryTests.cs new file mode 100644 index 00000000..511bb696 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Health/PerPlatformProbeWatcherRegistryTests.cs @@ -0,0 +1,112 @@ +using System.Runtime.CompilerServices; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Health; + +/// +/// Review 2026-10-21 G-4: the watcher's probe handles must live in the driver's +/// under the reserved id, follow the platform set, be +/// rebound on , and leave on stop. +/// +public sealed class PerPlatformProbeWatcherRegistryTests +{ + private sealed class FakeSubscriber : IGalaxySubscriber + { + private int _next; + public List Unsubscribes { get; } = []; + + public Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + => Task.FromResult>( + [.. fullReferences.Select(r => new SubscribeResult { TagAddress = r, ItemHandle = ++_next, WasSuccessful = true })]); + + public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) + { + Unsubscribes.Add([.. itemHandles]); + return Task.CompletedTask; + } + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Delay(Timeout.Infinite, cancellationToken); + yield break; + } + } + + private static (long Id, string Ref)[] Resolve(SubscriptionRegistry registry, int handle) => + [.. registry.ResolveSubscribers(handle)]; + + /// Synced platforms resolve through the registry under the reserved id; removed ones stop resolving. + [Fact] + public async Task Sync_registers_probe_handles_under_the_reserved_id_and_tracks_removals() + { + var registry = new SubscriptionRegistry(); + using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry); + + await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None); + + registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue(); + var bindings = watcher.CurrentBindings; + bindings.Count.ShouldBe(2); + foreach (var b in bindings) + { + Resolve(registry, b.ItemHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, b.FullReference)]); + } + + var bHandle = bindings.Single(b => b.FullReference == "B.ScanState").ItemHandle; + await watcher.SyncPlatformsAsync(["A"], CancellationToken.None); + Resolve(registry, bHandle).ShouldBeEmpty("a removed platform's handle must leave the fan-out map"); + var aHandle = bindings.Single(b => b.FullReference == "A.ScanState").ItemHandle; + Resolve(registry, aHandle).ShouldHaveSingleItem("positive control: the kept platform still resolves"); + } + + /// Resubscribe replaces the handles and rebinds: old handles stop resolving, new ones resolve. + [Fact] + public async Task Resubscribe_rebinds_the_registry_to_the_fresh_handles() + { + var registry = new SubscriptionRegistry(); + using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator(), registry); + await watcher.SyncPlatformsAsync(["A"], CancellationToken.None); + var oldHandle = watcher.CurrentBindings.Single().ItemHandle; + + (await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1); + + var newHandle = watcher.CurrentBindings.Single().ItemHandle; + newHandle.ShouldNotBe(oldHandle); + Resolve(registry, oldHandle).ShouldBeEmpty(); + Resolve(registry, newHandle).ShouldBe([(SubscriptionRegistry.ProbeSubscriptionId, "A.ScanState")]); + registry.TrackedSubscriptionCount.ShouldBe(1); + } + + /// StopAsync unsubscribes the live handles and removes the reserved entry. + [Fact] + public async Task StopAsync_unsubscribes_and_removes_the_reserved_entry() + { + var registry = new SubscriptionRegistry(); + var subscriber = new FakeSubscriber(); + var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry); + await watcher.SyncPlatformsAsync(["A", "B"], CancellationToken.None); + var handles = watcher.CurrentBindings.Select(b => b.ItemHandle).OrderBy(h => h).ToArray(); + + await watcher.StopAsync(CancellationToken.None); + + subscriber.Unsubscribes.ShouldHaveSingleItem().OrderBy(h => h).ShouldBe(handles); + registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse(); + registry.TrackedItemHandleCount.ShouldBe(0); + await Should.NotThrowAsync(() => watcher.StopAsync(CancellationToken.None)); // idempotent + } + + /// A watcher without a registry (the decoding-table unit tests) still works and registers nothing. + [Fact] + public async Task Without_a_registry_the_watcher_keeps_its_handles_private() + { + using var watcher = new PerPlatformProbeWatcher(new FakeSubscriber(), new HostStatusAggregator()); + await watcher.SyncPlatformsAsync(["A"], CancellationToken.None); + watcher.CurrentBindings.ShouldHaveSingleItem(); + (await watcher.ResubscribeAsync(CancellationToken.None)).ShouldBe(1); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ProcessWideTelemetryCollection.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ProcessWideTelemetryCollection.cs new file mode 100644 index 00000000..9d2f16fe --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ProcessWideTelemetryCollection.cs @@ -0,0 +1,17 @@ +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests; + +/// +/// Classes that assert on process-wide telemetry (an ActivityListener on the Galaxy +/// ActivitySource, a MeterListener on the Galaxy meter) run with parallelization disabled: +/// those listeners see every Galaxy span / measurement in the process, so any parallel class that +/// drives a traced subscriber or the data writer — including a driver's background probe seed that +/// outlives its test — pollutes their counts (F-FU6, observed flaky in lane F fix round 2). +/// +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class ProcessWideTelemetryCollection +{ + /// The collection name. + public const string Name = "Process-wide Galaxy telemetry"; +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpDispatchFaultTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpDispatchFaultTests.cs new file mode 100644 index 00000000..8d48d6d5 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/EventPumpDispatchFaultTests.cs @@ -0,0 +1,96 @@ +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Google.Protobuf.WellKnownTypes; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Review 2026-10-21 G-10: a dispatch-loop exit left the read loop draining into a channel +/// nobody read, with health Healthy and nothing telling the supervisor — a clean termination +/// evading the watchdog. The exit now reports a stream fault. The fault is injected through +/// the pump's handle factory, the one dispatch-side seam that runs outside the per-handler +/// isolation. +/// +public sealed class EventPumpDispatchFaultTests +{ + private const int WaitMs = 10_000; + + private sealed class ChannelSubscriber : IGalaxySubscriber + { + private readonly Channel _events = Channel.CreateUnbounded(); + + public Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + => Task.FromResult>([]); + + public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) => Task.CompletedTask; + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + + public ValueTask EmitAsync(int handle) => _events.Writer.WriteAsync(new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = handle, + Value = new MxValue { DoubleValue = 1 }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }); + } + + /// A dispatch-loop exception reports a stream fault carrying that exception. + [Fact] + public async Task Dispatch_loop_exit_reports_a_stream_fault() + { + var subscriber = new ChannelSubscriber(); + var registry = new SubscriptionRegistry(); + registry.Register(1, [new TagBinding("Tank01.Level", 7)]); + var fault = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await using var pump = new EventPump( + subscriber, registry, + handleFactory: _ => throw new InvalidDataException("synthetic dispatch failure"), + channelCapacity: 4, clientName: "DispatchFault", + onStreamFault: ex => fault.TrySetResult(ex)); + pump.Start(); + + await subscriber.EmitAsync(7); + + (await Task.WhenAny(fault.Task, Task.Delay(WaitMs))).ShouldBe(fault.Task, + "a dispatch-loop exit must be reported as a stream fault"); + (await fault.Task).ShouldBeOfType(); + } + + /// Positive control: the same event with a working handle factory dispatches and reports no fault. + [Fact] + public async Task Same_event_with_a_working_factory_dispatches_without_a_fault() + { + var subscriber = new ChannelSubscriber(); + var registry = new SubscriptionRegistry(); + registry.Register(1, [new TagBinding("Tank01.Level", 7)]); + var faulted = false; + var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await using var pump = new EventPump( + subscriber, registry, channelCapacity: 4, clientName: "DispatchFault", + onStreamFault: _ => faulted = true); + pump.OnDataChange += (_, e) => delivered.TrySetResult(e); + pump.Start(); + + await subscriber.EmitAsync(7); + + (await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task); + (await delivered.Task).FullReference.ShouldBe("Tank01.Level"); + faulted.ShouldBeFalse(); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs new file mode 100644 index 00000000..9876c09f --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound1Tests.cs @@ -0,0 +1,406 @@ +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Google.Protobuf.WellKnownTypes; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Lane F fix round 1 (review of #735): the probe watcher is seeded at InitializeAsync +/// and re-seeded on rediscovery (the runtime never calls DiscoverAsync on a live driver); +/// a SubscribeAsync landing mid-teardown cannot create a pump on the dying subscriber; a +/// gateway close that never answers cannot park ShutdownAsync; a re-subscribe that +/// straddles a teardown never rebinds into the next runtime's registry. +/// +public sealed class GalaxyDriverFixRound1Tests +{ + internal const int WaitMs = 10_000; + + internal static GalaxyDriverOptions Opts(bool watchDeploy = false) => new( + new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), + new GalaxyMxAccessOptions("FixRound1"), + new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy), + new GalaxyReconnectOptions()); + + // ───────────────────────────── fakes ───────────────────────────── + + private sealed class ScriptedPlatformSource : IGalaxyPlatformSource + { + private int _calls; + public int Calls => Volatile.Read(ref _calls); + public Func> Script { get; init; } = _ => ["Plat01"]; + + public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + => Task.FromResult(Script(Interlocked.Increment(ref _calls))); + } + + internal sealed class ParkableSubscriber : IGalaxySubscriber + { + private readonly Channel _events = Channel.CreateUnbounded(); + private int _nextHandle; + private int _subscribeCalls; + + public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase); + public List Subscribed { get; } = []; + public int SubscribeCalls => Volatile.Read(ref _subscribeCalls); + + /// When set, UnsubscribeBulk parks until . + public bool ParkUnsubscribe { get; set; } + public TaskCompletionSource UnsubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _releaseUnsubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void ReleaseUnsubscribe() => _releaseUnsubscribe.TrySetResult(); + + /// When > 0, that SubscribeBulk call (1-based) parks until . + public int ParkSubscribeCall { get; set; } + public TaskCompletionSource SubscribeEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _releaseSubscribe = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void ReleaseSubscribe() => _releaseSubscribe.TrySetResult(); + + public async Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + { + var call = Interlocked.Increment(ref _subscribeCalls); + if (call == ParkSubscribeCall) + { + SubscribeEntered.TrySetResult(); + await _releaseSubscribe.Task.WaitAsync(cancellationToken); + } + var results = new List(); + lock (Subscribed) + { + foreach (var r in fullReferences) + { + var h = Interlocked.Increment(ref _nextHandle); + Subscribed.Add(r); + HandleByRef[r] = h; + results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true }); + } + } + return results; + } + + /// Every handle passed to UnsubscribeBulk, in call order. + public List Unsubscribed { get; } = []; + + public async Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) + { + lock (Unsubscribed) Unsubscribed.AddRange(itemHandles); + if (ParkUnsubscribe) + { + UnsubscribeEntered.TrySetResult(); + await _releaseUnsubscribe.Task.WaitAsync(cancellationToken); + } + } + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + + public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = handle, + Value = new MxValue { BoolValue = value }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }); + } + + internal sealed class ChannelDeployWatchSource : IGalaxyDeployWatchSource + { + public Channel Events { get; } = Channel.CreateUnbounded(); + public int Opened; + + public async IAsyncEnumerable WatchAsync( + DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken) + { + Interlocked.Increment(ref Opened); + await foreach (var ev in Events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + + public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Events.Writer.WriteAsync(new DeployEvent + { + Sequence = sequence, + TimeOfLastDeployPresent = true, + TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime), + }); + } + + internal static async Task WaitUntilAsync(Func condition, string because) + { + var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because); + await Task.Delay(10); + } + } + + internal static HostState? StateOf(GalaxyDriver driver, string host) => + driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State; + + // ───────────────────────────── IMPORTANT 1: seed at init, re-seed on rediscovery ───────────────────────────── + + /// A fresh InitializeAsync with NO DiscoverAsync yields per-platform host statuses. + [Fact] + public async Task InitializeAsync_seeds_the_probe_set_and_ScanState_updates_host_status_without_DiscoverAsync() + { + var sub = new ParkableSubscriber(); + var platforms = new ScriptedPlatformSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + + await driver.InitializeAsync("{}", CancellationToken.None); + + // The seed runs in the background (fix round 2, N1) — init does not wait on it. + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "the probe watcher must be seeded after init"); + platforms.Calls.ShouldBe(1); + lock (sub.Subscribed) sub.Subscribed.ShouldContain("Plat01.ScanState"); + driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]); + + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump + await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true); + await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running, + "a ScanState event must surface the seeded platform on GetHostStatuses"); + } + + /// A failed seed does not fail init; the next deploy-watcher rediscovery re-seeds. + [Fact] + public async Task Failed_seed_does_not_fail_init_and_rediscovery_reseeds() + { + var sub = new ParkableSubscriber(); + var deploys = new ChannelDeployWatchSource(); + var platforms = new ScriptedPlatformSource + { + Script = call => call == 1 ? throw new IOException("repository unavailable") : ["Plat01", "Eng01"], + }; + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null, + subscriber: sub, platformSource: platforms, deployWatchSource: deploys); + var raised = 0; + driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised); + + await driver.InitializeAsync("{}", CancellationToken.None); + driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a platform-seed failure must not fail init"); + await WaitUntilAsync(() => platforms.Calls == 1, "the (background) init seed must have run"); + sub.Subscribed.ShouldBeEmpty(); + + await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open"); + await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap (suppressed) + await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); // a real deploy + + await WaitUntilAsync(() => Volatile.Read(ref raised) == 1, "positive control: the deploy must raise rediscovery"); + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2, + "the rediscovery must re-seed the probe set"); + platforms.Calls.ShouldBe(2); + sub.Subscribed.ShouldBe(["Plat01.ScanState", "Eng01.ScanState"], ignoreOrder: true); + } + + private sealed class ParkingPlatformSource : IGalaxyPlatformSource + { + private int _calls; + public int Calls => Volatile.Read(ref _calls); + public int ParkCall { get; init; } + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _release.TrySetResult(); + + public async Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + { + var call = Interlocked.Increment(ref _calls); + if (call == ParkCall) + { + Entered.TrySetResult(); + await _release.Task.WaitAsync(cancellationToken); + return ["Plat01"]; // the pre-deploy view this pass fetched + } + return call == 1 ? [] : ["Plat01", "Eng02"]; + } + } + + /// + /// A deploy that lands while a re-seed pass is mid-fetch gets a pass of its own that starts + /// after it — otherwise the probe set would stay on the stale view until the next deploy. + /// + [Fact] + public async Task A_deploy_during_an_in_flight_reseed_gets_its_own_pass() + { + var sub = new ParkableSubscriber(); + var deploys = new ChannelDeployWatchSource(); + var platforms = new ParkingPlatformSource { ParkCall = 2 }; // call 1 = init seed, call 2 = first rediscovery + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: true), hierarchySource: null, + subscriber: sub, platformSource: platforms, deployWatchSource: deploys); + var raised = 0; + driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised); + try + { + await driver.InitializeAsync("{}", CancellationToken.None); + await WaitUntilAsync(() => Volatile.Read(ref deploys.Opened) == 1, "the deploy watch must open"); + await deploys.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); // bootstrap + await deploys.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); + (await Task.WhenAny(platforms.Entered.Task, Task.Delay(WaitMs))).ShouldBe(platforms.Entered.Task, + "the first rediscovery pass must be parked mid-fetch"); + + await deploys.PushAsync(3, new DateTimeOffset(2026, 1, 3, 0, 0, 0, TimeSpan.Zero)); + await WaitUntilAsync(() => Volatile.Read(ref raised) == 2, "positive control: the second deploy raised rediscovery"); + } + finally + { + platforms.Release(); + } + + await WaitUntilAsync(() => platforms.Calls == 3, "the deploy that landed mid-pass must get its own re-seed"); + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 2, + "the final probe set must reflect the latest deploy, not the parked pass's stale view"); + } + + /// The production path (owned runtime) queries the platform source at init, with no DiscoverAsync. + [Fact] + public async Task Production_path_queries_the_platform_source_at_init() + { + var platforms = new ScriptedPlatformSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: platforms) + { + SessionOpenOverrideForTests = _ => Task.CompletedTask, + }; + + await driver.InitializeAsync("{}", CancellationToken.None); + + await WaitUntilAsync(() => platforms.Calls == 1, "the owned runtime must seed its probe set after init"); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + // ───────────────────────────── IMPORTANT 2: no pump on a dying subscriber ───────────────────────────── + + /// + /// A SubscribeAsync that lands while the teardown is parked (after the stopping flag, before + /// the subscriber is released) is refused and leaves no pump; before the shutdown the same + /// call works (positive control). + /// + [Fact] + public async Task SubscribeAsync_during_a_teardown_is_refused_and_leaves_no_pump() + { + var sub = new ParkableSubscriber(); + var platforms = new ScriptedPlatformSource(); // gives the probe watcher a handle to unsubscribe + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + await driver.InitializeAsync("{}", CancellationToken.None); + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "the (background) seed must give the probe watcher its handle first"); + + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime"); + + sub.ParkUnsubscribe = true; + var shutdown = driver.ShutdownAsync(CancellationToken.None); + var subscribeCallsBefore = sub.SubscribeCalls; + try + { + (await Task.WhenAny(sub.UnsubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.UnsubscribeEntered.Task, + "the teardown must be parked inside the probe watcher's unsubscribe"); + shutdown.IsCompleted.ShouldBeFalse(); + + // The pump is still live here (the teardown takes it after the probe watcher) — reusing it + // is exactly the race: the call must be refused, not served on the dying subscriber. + driver.HasEventPumpForTests.ShouldBeTrue("the interleaving under test: the old pump is not yet taken"); + await Should.ThrowAsync( + () => driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None)); + sub.SubscribeCalls.ShouldBe(subscribeCallsBefore, "a refused subscribe must not reach the gateway"); + } + finally + { + // Never leave the teardown parked: it holds the lifecycle gate the await-using dispose needs. + sub.ReleaseUnsubscribe(); + } + await shutdown.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + driver.HasEventPumpForTests.ShouldBeFalse("no pump may survive the teardown"); + driver.TrackedSubscriptionCountForTests.ShouldBe(0); + + await driver.InitializeAsync("{}", CancellationToken.None); + driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump"); + } + + // ───────────────────────────── IMPORTANT 3: bounded gateway close ───────────────────────────── + + /// A gateway CloseSession that never answers cannot park ShutdownAsync past the close deadline. + [Fact] + public async Task ShutdownAsync_is_bounded_when_the_gateway_close_never_answers() + { + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: new ScriptedPlatformSource()) + { + SessionOpenOverrideForTests = _ => Task.CompletedTask, + SessionCloseDeadline = TimeSpan.FromMilliseconds(300), + }; + await driver.InitializeAsync("{}", CancellationToken.None); + var closeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var closeToken = CancellationToken.None; + driver.OwnedMxSessionForTests.ShouldNotBeNull().CloseOverrideForTests = async ct => + { + closeToken = ct; + closeEntered.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); // a blackholed gateway + }; + + var sw = System.Diagnostics.Stopwatch.StartNew(); + await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + sw.Stop(); + + closeEntered.Task.IsCompleted.ShouldBeTrue("positive control: the close was attempted"); + closeToken.IsCancellationRequested.ShouldBeTrue("the close must have been cancelled by the deadline"); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "ShutdownAsync must return at the close deadline, not the SDK's 30 s"); + driver.OwnedMxSessionForTests.ShouldBeNull(); + driver.GetHealth().State.ShouldBe(DriverState.Unknown); + } + + // ───────────────────────────── MINOR: generation guard on a straddling re-subscribe ───────────────────────────── + + /// + /// A replay whose SubscribeBulk completes after a teardown must not Rebind into the cleared + /// registry; the same replay without a teardown rebinds (positive control). + /// + [Fact] + public async Task A_resubscribe_that_straddles_a_teardown_does_not_rebind() + { + // Positive control first. + { + var sub = new ParkableSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + await driver.InvokeReplayForTestAsync(CancellationToken.None); + driver.TrackedSubscriptionCountForTests.ShouldBe(1, "a replay on a live runtime rebinds the subscription"); + sub.SubscribeCalls.ShouldBe(2); + } + + { + var sub = new ParkableSubscriber { ParkSubscribeCall = 2 }; + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + + var replay = driver.InvokeReplayForTestAsync(CancellationToken.None); + (await Task.WhenAny(sub.SubscribeEntered.Task, Task.Delay(WaitMs))).ShouldBe(sub.SubscribeEntered.Task, + "the replay must be parked inside its SubscribeBulk"); + + await driver.ShutdownAsync(CancellationToken.None); + driver.TrackedSubscriptionCountForTests.ShouldBe(0); + + sub.ReleaseSubscribe(); + await replay.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + driver.TrackedSubscriptionCountForTests.ShouldBe(0, "a Rebind after the teardown would resurrect dead handles"); + } + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound2Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound2Tests.cs new file mode 100644 index 00000000..ddd97023 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverFixRound2Tests.cs @@ -0,0 +1,313 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; +using static ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime.GalaxyDriverFixRound1Tests; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Lane F fix round 2 (re-review of #735): +/// +/// N1 — the init platform seed runs in the background: a slow or cancelled repository +/// fetch can neither delay nor fail InitializeAsync (Galaxy's Tier-A Initialize budget is +/// 5 s, and a cold gateway cache can spend all of it), and a seed that lands after a teardown +/// is discarded. +/// N2 — a seed requested under a new runtime always gets a pass of its own, even when a +/// worker from the previous runtime is still running. +/// N3 — a subscribe / subscribe-once read whose SubscribeBulk straddles a teardown fails +/// and unsubscribes instead of registering into the next runtime's registry. +/// The probe watcher's own subscribe re-checks its stop after the await. +/// +/// +public sealed class GalaxyDriverFixRound2Tests +{ + private static readonly TimeSpan InitBudget = TimeSpan.FromSeconds(2); + + /// A platform source whose fetches park until released (optionally ignoring cancellation). + private sealed class GatedPlatformSource : IGalaxyPlatformSource + { + private int _calls; + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int Calls => Volatile.Read(ref _calls); + public int ParkCall { get; init; } = 1; + public bool IgnoreCancellation { get; init; } + public Func> Result { get; init; } = _ => ["Plat01"]; + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ParkedToken { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _release.TrySetResult(); + + public async Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + { + var call = Interlocked.Increment(ref _calls); + if (call == ParkCall) + { + ParkedToken.TrySetResult(cancellationToken); + Entered.TrySetResult(); + if (IgnoreCancellation) await _release.Task; + else await _release.Task.WaitAsync(cancellationToken); + } + return Result(call); + } + } + + private sealed class ThrowingPlatformSource(Exception ex) : IGalaxyPlatformSource + { + private int _calls; + public int Calls => Volatile.Read(ref _calls); + + public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _calls); + return Task.FromException>(ex); + } + } + + private static async Task Entered(TaskCompletionSource tcs, string because) => + (await Task.WhenAny(tcs.Task, Task.Delay(WaitMs))).ShouldBe(tcs.Task, because); + + // ───────────────────────────── N1: background init seed ───────────────────────────── + + /// + /// A repository fetch that never completes cannot hold InitializeAsync: init returns Healthy + /// inside its budget, and the teardown cancels the parked fetch. + /// + [Fact] + public async Task InitializeAsync_returns_Healthy_while_the_platform_fetch_never_completes() + { + var sub = new ParkableSubscriber(); + var platforms = new GatedPlatformSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + try + { + var sw = System.Diagnostics.Stopwatch.StartNew(); + await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); + sw.Stop(); + + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + await Entered(platforms.Entered, "positive control: the seed fetch was started"); + sw.Elapsed.ShouldBeLessThan(InitBudget); + + await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + var token = await platforms.ParkedToken.Task; + token.IsCancellationRequested.ShouldBeTrue("the teardown must cancel a seed still in flight"); + } + finally + { + platforms.Release(); + } + } + + /// A fetch that is cancelled (the gateway's own deadline) does not fail init. + [Fact] + public async Task A_cancelled_platform_fetch_does_not_fail_InitializeAsync() + { + var sub = new ParkableSubscriber(); + var platforms = new ThrowingPlatformSource(new OperationCanceledException("gateway deadline")); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + + await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); + + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + await WaitUntilAsync(() => platforms.Calls == 1, "positive control: the seed fetch ran"); + driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty(); + + // The driver keeps serving after the failed seed. + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + driver.TrackedSubscriptionCountForTests.ShouldBe(1); + } + + /// + /// The production failure mode: the host's Tier-A Initialize timeout cancels the init token + /// while the (cold-cache) fetch is still running. Init must already have returned Healthy, and + /// the seed — tied to the runtime, not to the init call — must survive that cancellation and + /// still land. + /// + [Fact] + public async Task Init_token_cancellation_mid_fetch_neither_fails_init_nor_cancels_the_seed() + { + var sub = new ParkableSubscriber(); + var platforms = new GatedPlatformSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + using var initCts = new CancellationTokenSource(); + try + { + initCts.CancelAfter(TimeSpan.FromMilliseconds(300)); // the Polly Initialize timeout, shrunk + await driver.InitializeAsync("{}", initCts.Token).WaitAsync(InitBudget); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + + await Entered(platforms.Entered, "the seed fetch must be in flight"); + await WaitUntilAsync(() => initCts.IsCancellationRequested, "the init token fires while the fetch is parked"); + (await platforms.ParkedToken.Task).IsCancellationRequested.ShouldBeFalse( + "the seed belongs to the runtime; the init call's timeout must not cancel it"); + } + finally + { + platforms.Release(); + } + + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "the seed must still land after the init token was cancelled"); + } + + /// Positive control: a fetch that completes after init still populates host statuses. + [Fact] + public async Task A_fetch_completing_after_init_populates_host_statuses() + { + var sub = new ParkableSubscriber(); + var platforms = new GatedPlatformSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + try + { + await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); + await Entered(platforms.Entered, "the seed must be parked"); + driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBeEmpty(); + } + finally + { + platforms.Release(); + } + + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "the late seed must reach the probe watcher"); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // starts the pump + await sub.EmitBoolAsync(sub.HandleByRef["Plat01.ScanState"], true); + await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running, + "a ScanState event must surface the late-seeded platform"); + } + + // ───────────────────────────── N1 generation + N2: straddling worker ───────────────────────────── + + /// + /// A seed from the previous runtime that completes after a teardown + re-init is discarded, + /// and the new runtime's own seed request still gets its own pass. + /// + [Fact] + public async Task A_seed_straddling_a_teardown_is_discarded_and_the_new_runtime_is_seeded() + { + var sub = new ParkableSubscriber(); + var platforms = new GatedPlatformSource + { + IgnoreCancellation = true, // a fetch that finishes regardless — only the generation check can stop it + Result = call => call == 1 ? ["Stale01"] : ["Plat01"], + }; + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, platformSource: platforms); + try + { + await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); + await Entered(platforms.Entered, "runtime 1's seed must be parked"); + + await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + await driver.InitializeAsync("{}", CancellationToken.None).WaitAsync(InitBudget); + } + finally + { + platforms.Release(); + } + + await WaitUntilAsync(() => platforms.Calls == 2, "the request made under the new runtime must get its own pass"); + await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1, + "the new runtime must be seeded"); + driver.ProbeWatcherForTests!.WatchedPlatforms.ShouldBe(["Plat01"]); + lock (sub.Subscribed) sub.Subscribed.ShouldNotContain("Stale01.ScanState", "runtime 1's late result must be discarded"); + } + + // ───────────────────────────── probe watcher: re-check after the SubscribeBulk await ───────────────────────────── + + /// + /// A watcher sync whose SubscribeBulk completes after StopAsync must not write the probe entry + /// back into the registry, and must release the handles it just acquired. Positive control: + /// the same sync without a stop registers. + /// + [Fact] + public async Task Watcher_sync_straddling_a_stop_does_not_resurrect_the_probe_entry() + { + { + var registry = new SubscriptionRegistry(); + using var watcher = new PerPlatformProbeWatcher(new ParkableSubscriber(), new HostStatusAggregator(), registry); + await watcher.SyncPlatformsAsync(["A"], CancellationToken.None); + registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeTrue("positive control"); + } + + { + var registry = new SubscriptionRegistry(); + var sub = new ParkableSubscriber { ParkSubscribeCall = 1 }; + var watcher = new PerPlatformProbeWatcher(sub, new HostStatusAggregator(), registry); + + var sync = watcher.SyncPlatformsAsync(["A"], CancellationToken.None); + await Entered(sub.SubscribeEntered, "the sync must be parked in SubscribeBulk"); + await watcher.StopAsync(CancellationToken.None); + + sub.ReleaseSubscribe(); + await sync.WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + registry.Contains(SubscriptionRegistry.ProbeSubscriptionId).ShouldBeFalse("a stopped watcher must not re-register"); + watcher.WatchedPlatforms.ShouldBeEmpty(); + lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["A.ScanState"], + "the handle acquired after the stop must be released"); + } + } + + // ───────────────────────────── N3: subscribe / read straddling a teardown ───────────────────────────── + + /// + /// A SubscribeAsync whose SubscribeBulk completes after the teardown fails, registers nothing + /// and unsubscribes its handles; a subscribe on the live runtime registers (positive control). + /// + [Fact] + public async Task SubscribeAsync_straddling_a_teardown_fails_and_registers_nothing() + { + var sub = new ParkableSubscriber { ParkSubscribeCall = 2 }; + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + driver.TrackedSubscriptionCountForTests.ShouldBe(1, "positive control: a live-runtime subscribe registers"); + + var straddling = driver.SubscribeAsync(["Tank02.Level"], TimeSpan.Zero, CancellationToken.None); + try + { + await Entered(sub.SubscribeEntered, "the subscribe must be parked in SubscribeBulk"); + await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + driver.TrackedSubscriptionCountForTests.ShouldBe(0); + } + finally + { + sub.ReleaseSubscribe(); + } + + await Should.ThrowAsync(() => straddling.WaitAsync(TimeSpan.FromMilliseconds(WaitMs))); + driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the straddling subscribe must not land in the registry"); + lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank02.Level"]); + + await driver.InitializeAsync("{}", CancellationToken.None); + driver.TrackedSubscriptionCountForTests.ShouldBe(0, "the next runtime starts with an empty registry"); + } + + /// The subscribe-once read path: same straddle, same outcome. + [Fact] + public async Task Subscribe_once_read_straddling_a_teardown_fails_and_registers_nothing() + { + var sub = new ParkableSubscriber { ParkSubscribeCall = 1 }; + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + + var read = driver.ReadAsync(["Tank01.Level"], CancellationToken.None); + try + { + await Entered(sub.SubscribeEntered, "the read must be parked in SubscribeBulk"); + await driver.ShutdownAsync(CancellationToken.None).WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + } + finally + { + sub.ReleaseSubscribe(); + } + + await Should.ThrowAsync(() => read.WaitAsync(TimeSpan.FromMilliseconds(WaitMs))); + driver.TrackedSubscriptionCountForTests.ShouldBe(0); + lock (sub.Unsubscribed) sub.Unsubscribed.ShouldContain(sub.HandleByRef["Tank01.Level"]); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverProbeWiringTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverProbeWiringTests.cs new file mode 100644 index 00000000..af842c93 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverProbeWiringTests.cs @@ -0,0 +1,217 @@ +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Google.Protobuf.WellKnownTypes; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Review 2026-10-21 G-4 / G-7 / G-11: the per-platform ScanState probing was structurally +/// dead (its handles were never in the registry the pump resolves through, and a reconnect +/// never re-subscribed them), discovery fetched the full galaxy twice, and +/// LastSuccessfulRead never moved on data. These drive the real driver → pump → watcher +/// → aggregator path with a fake subscriber and a fake hierarchy. +/// +public sealed class GalaxyDriverProbeWiringTests +{ + private const int WaitMs = 10_000; + + private static GalaxyDriverOptions Opts() => new( + new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), + new GalaxyMxAccessOptions("ProbeWiring"), + new GalaxyRepositoryOptions(WatchDeployEvents: false), + new GalaxyReconnectOptions()); + + private sealed class CountingHierarchySource : IGalaxyHierarchySource + { + public int Calls; + + public Task> GetHierarchyAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref Calls); + var platform = new GalaxyObject { GobjectId = 1, TagName = "Plat01", ContainedName = "Plat01" }; + platform.TemplateChain.Add("$WinPlatform"); + var tank = new GalaxyObject { GobjectId = 2, TagName = "Tank01", ContainedName = "Tank01", ParentGobjectId = 1 }; + tank.TemplateChain.Add("$UserDefined"); + tank.Attributes.Add(new GalaxyAttribute { AttributeName = "Level", FullTagReference = "Tank01.Level", MxDataType = 5 }); + return Task.FromResult>([platform, tank]); + } + } + + private sealed class StreamingSubscriber : IGalaxySubscriber + { + private readonly Channel _events = Channel.CreateUnbounded(); + private int _nextHandle; + + public List Subscribed { get; } = []; + public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase); + + public Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + { + var results = new List(); + lock (Subscribed) + { + foreach (var r in fullReferences) + { + var h = Interlocked.Increment(ref _nextHandle); + Subscribed.Add(r); + HandleByRef[r] = h; + results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true }); + } + } + return Task.FromResult>(results); + } + + public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) => Task.CompletedTask; + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + + public ValueTask EmitBoolAsync(int handle, bool value) => _events.Writer.WriteAsync(new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = handle, + Value = new MxValue { BoolValue = value }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }); + + public ValueTask EmitDoubleAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = handle, + Value = new MxValue { DoubleValue = value }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }); + } + + private sealed class NullBuilder : IAddressSpaceBuilder + { + public IAddressSpaceBuilder Folder(string browseName, string displayName) => this; + public IVariableHandle Variable(string browseName, string displayName, DriverAttributeInfo attributeInfo) => new Handle(attributeInfo.FullName); + public void AddProperty(string browseName, DriverDataType dataType, object? value) { } + private sealed class Handle(string fullReference) : IVariableHandle + { + public string FullReference { get; } = fullReference; + public IAlarmConditionSink MarkAsAlarmCondition(AlarmConditionInfo alarmInfo) => new NoopSink(); + } + private sealed class NoopSink : IAlarmConditionSink + { + public void OnTransition(AlarmEventArgs args) { } + } + } + + private static async Task WaitUntilAsync(Func condition, string because) + { + var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because); + await Task.Delay(10); + } + } + + private static HostState? StateOf(GalaxyDriver driver, string host) => + driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State; + + /// + /// A ScanState event on the probe handle updates GetHostStatuses; after a reconnect replay + /// the probe is re-subscribed and events on the NEW handle keep updating it, while the dead + /// pre-reconnect handle no longer does. Probe traffic never reaches public OnDataChange. + /// + [Fact] + public async Task ScanState_event_updates_GetHostStatuses_and_survives_a_reconnect() + { + var sub = new StreamingSubscriber(); + var source = new CountingHierarchySource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub); + var publicRefs = new List(); + driver.OnDataChange += (_, e) => { lock (publicRefs) publicRefs.Add(e.FullReference); }; + + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None); + // The pump starts on the first value subscription (the probe watcher does not start it). + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + + var probeHandle = sub.HandleByRef["Plat01.ScanState"]; + await sub.EmitBoolAsync(probeHandle, true); + await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Running, + "a ScanState=true event must surface Plat01 as Running"); + + // Reconnect replay: every pre-reconnect handle is dead; the probe must rebind. + await driver.InvokeReplayForTestAsync(CancellationToken.None); + var rebound = sub.HandleByRef["Plat01.ScanState"]; + rebound.ShouldNotBe(probeHandle, "replay must re-subscribe the probe and record the fresh handle"); + + await sub.EmitBoolAsync(rebound, false); + await WaitUntilAsync(() => StateOf(driver, "Plat01") == HostState.Stopped, + "a ScanState=false event on the post-reconnect handle must surface Plat01 as Stopped"); + + // The dead handle is no longer routed (positive control: the live one still is). + await sub.EmitBoolAsync(probeHandle, true); + await sub.EmitDoubleAsync(sub.HandleByRef["Tank01.Level"], 1.0); + await WaitUntilAsync(() => { lock (publicRefs) return publicRefs.Contains("Tank01.Level"); }, "value traffic must still flow"); + StateOf(driver, "Plat01").ShouldBe(HostState.Stopped, "an event on the pre-reconnect handle must be dropped"); + + lock (publicRefs) publicRefs.ShouldNotContain("Plat01.ScanState", "probe traffic is the watcher's, never the host's"); + } + + /// Discovery asks the gateway for the hierarchy once; the probe set is selected from that same sweep. + [Fact] + public async Task DiscoverAsync_fetches_the_hierarchy_once_and_still_syncs_the_probe_set() + { + var sub = new StreamingSubscriber(); + var source = new CountingHierarchySource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: source, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + + await driver.DiscoverAsync(new NullBuilder(), CancellationToken.None); + + source.Calls.ShouldBe(1, "the probe selection must reuse the discoverer's sweep"); + sub.Subscribed.ShouldContain("Plat01.ScanState", "positive control: the platform probe was still synced"); + driver.ProbeWatcherForTests.ShouldNotBeNull().WatchedPlatforms.ShouldBe(["Plat01"]); + } + + /// Inbound stream data advances LastSuccessfulRead (throttled); the throttle holds the stamp still. + [Fact] + public async Task Stream_data_advances_LastSuccessfulRead_under_a_throttle() + { + var sub = new StreamingSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub) + { + LivenessStampThrottle = TimeSpan.FromHours(1), + }; + await driver.InitializeAsync("{}", CancellationToken.None); + var initStamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); + await Task.Delay(20); + + var delivered = 0; + driver.OnDataChange += (_, _) => Interlocked.Increment(ref delivered); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + var handle = sub.HandleByRef["Tank01.Level"]; + + await sub.EmitDoubleAsync(handle, 1.0); + await WaitUntilAsync(() => Volatile.Read(ref delivered) == 1, "first event must be delivered"); + var first = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); + first.ShouldBeGreaterThan(initStamp, "data on the stream is liveness evidence"); + + await Task.Delay(20); + await sub.EmitDoubleAsync(handle, 2.0); + await WaitUntilAsync(() => Volatile.Read(ref delivered) == 2, "second event must be delivered"); + driver.GetHealth().LastSuccessfulRead.ShouldBe(first, "inside the throttle window the stamp holds still"); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs new file mode 100644 index 00000000..cdb2ec69 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyDriverShutdownTests.cs @@ -0,0 +1,386 @@ +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Google.Protobuf.WellKnownTypes; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// used to be a no-op (review 2026-10-21, G-1/G-2/G-9): +/// the gRPC runtime outlived the driver on every respawn, removal and /raw browse, and +/// the host's comms-loss recovery (ShutdownAsync → InitializeAsync on the SAME +/// object) neither tore down nor rebuilt anything. These tests pin the teardown at every +/// seam a fake can reach — the StreamEvents stream, the deploy-watch stream, the alarm feed, +/// the owned channels + session — and the rebuild on the next InitializeAsync. +/// +public sealed class GalaxyDriverShutdownTests +{ + private const int WaitMs = 10_000; + + private static GalaxyDriverOptions Opts(bool watchDeploy = true) => new( + new GalaxyGatewayOptions("https://mxgw.test:5001", "key"), + new GalaxyMxAccessOptions("ShutdownTest"), + new GalaxyRepositoryOptions(WatchDeployEvents: watchDeploy), + new GalaxyReconnectOptions()); + + // ───────────────────────────── fakes ───────────────────────────── + + /// + /// Subscriber whose event stream records the token it was opened with and completes a + /// signal when that token fires — the observable for "the StreamEvents stream was cancelled". + /// + private sealed class StreamingSubscriber : IGalaxySubscriber + { + private readonly Channel _events = Channel.CreateUnbounded(); + private int _nextHandle; + + public Dictionary HandleByRef { get; } = new(StringComparer.OrdinalIgnoreCase); + public List Unsubscribed { get; } = []; + public int StreamsOpened; + public TaskCompletionSource StreamCancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task> SubscribeBulkAsync( + IReadOnlyList fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken) + { + var results = new List(); + foreach (var r in fullReferences) + { + var h = Interlocked.Increment(ref _nextHandle); + HandleByRef[r] = h; + results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true }); + } + return Task.FromResult>(results); + } + + public Task UnsubscribeBulkAsync(IReadOnlyList itemHandles, CancellationToken cancellationToken) + { + lock (Unsubscribed) Unsubscribed.AddRange(itemHandles); + return Task.CompletedTask; + } + + public async IAsyncEnumerable StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + Interlocked.Increment(ref StreamsOpened); + using var _ = cancellationToken.Register(() => StreamCancelled.TrySetResult()); + await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + + public ValueTask EmitAsync(int handle, double value) => _events.Writer.WriteAsync(new MxEvent + { + Family = MxEventFamily.OnDataChange, + ItemHandle = handle, + Value = new MxValue { DoubleValue = value }, + Quality = 192, + SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow), + }); + } + + /// + /// Deploy-watch source that counts OPEN enumerations (incremented on open, decremented when + /// the enumerator is cancelled or completes) — the observable for "exactly one watcher". + /// + private sealed class CountingDeployWatchSource : IGalaxyDeployWatchSource + { + private int _open; + private int _opened; + public Channel Current { get; private set; } = Channel.CreateUnbounded(); + + public int Open => Volatile.Read(ref _open); + public int Opened => Volatile.Read(ref _opened); + + public async IAsyncEnumerable WatchAsync( + DateTimeOffset? lastSeenDeployTime, [EnumeratorCancellation] CancellationToken cancellationToken) + { + Interlocked.Increment(ref _open); + Interlocked.Increment(ref _opened); + var channel = Current = Channel.CreateUnbounded(); + try + { + await foreach (var ev in channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + yield return ev; + } + } + finally + { + Interlocked.Decrement(ref _open); + } + } + + public ValueTask PushAsync(ulong sequence, DateTimeOffset deployTime) => Current.Writer.WriteAsync(new DeployEvent + { + Sequence = sequence, + TimeOfLastDeployPresent = true, + TimeOfLastDeploy = Timestamp.FromDateTimeOffset(deployTime), + }); + } + + /// Canned platform set so a production-path init never dials the (fake) gateway for one. + private sealed class NoPlatforms : IGalaxyPlatformSource + { + public Task> GetPlatformTagNamesAsync(CancellationToken cancellationToken) + => Task.FromResult>([]); + } + + private sealed class RecordingAlarmFeed : IGalaxyAlarmFeed + { + public int Starts; + public int Disposes; + public event EventHandler? OnAlarmTransition; + public void Start() => Starts++; + public void Emit() => OnAlarmTransition?.Invoke(this, new GalaxyAlarmTransition( + "Tank01.Level.HiHi", "Tank01", "AnalogLimitAlarm.HiHi", GalaxyAlarmTransitionKind.Raise, + AlarmSeverity.High, 800, 750, null, DateTime.UtcNow, "", "", "Process", "hi-hi")); + public ValueTask DisposeAsync() { Disposes++; return ValueTask.CompletedTask; } + } + + private static async Task WaitUntilAsync(Func condition, string because) + { + var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because); + await Task.Delay(10); + } + } + + // ───────────────────────────── G-1: streams + pump ───────────────────────────── + + /// + /// After ShutdownAsync the StreamEvents stream is cancelled, the pump is gone and the + /// registry is empty. Positive control: an event delivered through that same stream + /// reached OnDataChange before the shutdown. + /// + [Fact] + public async Task ShutdownAsync_cancels_the_event_stream_and_releases_the_pump() + { + var sub = new StreamingSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + + var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + driver.OnDataChange += (_, e) => delivered.TrySetResult(e); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + driver.HasEventPumpForTests.ShouldBeTrue(); + + await sub.EmitAsync(sub.HandleByRef["Tank01.Level"], 42.0); + (await Task.WhenAny(delivered.Task, Task.Delay(WaitMs))).ShouldBe(delivered.Task, + "positive control: the stream must deliver before the shutdown"); + + await driver.ShutdownAsync(CancellationToken.None); + + // Bounded wait, not an instantaneous IsCompleted check: the pump's dispose does not promise the + // enumerator's cancellation registration has run by the time it returns (observed flaky). + (await Task.WhenAny(sub.StreamCancelled.Task, Task.Delay(WaitMs))).ShouldBe(sub.StreamCancelled.Task, + "ShutdownAsync must cancel the StreamEvents stream"); + driver.HasEventPumpForTests.ShouldBeFalse(); + driver.TrackedSubscriptionCountForTests.ShouldBe(0, "handles died with the session; the host re-subscribes after re-init"); + driver.GetHealth().State.ShouldBe(DriverState.Unknown); + driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull("the liveness evidence survives a shutdown"); + } + + /// + /// An injected alarm feed is detached on shutdown (the injector owns it, so it is NOT + /// disposed) and re-wired by the next SubscribeAlarmsAsync after a re-init. Positive + /// control on both sides of the shutdown. + /// + [Fact] + public async Task ShutdownAsync_detaches_the_alarm_feed_and_reinit_rewires_it() + { + var feed = new RecordingAlarmFeed(); + var sub = new StreamingSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, + subscriber: sub, alarmFeed: feed); + var received = 0; + driver.OnAlarmEvent += (_, _) => Interlocked.Increment(ref received); + + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None); + feed.Emit(); + received.ShouldBe(1, "positive control: the feed is wired before the shutdown"); + + await driver.ShutdownAsync(CancellationToken.None); + feed.Emit(); + received.ShouldBe(1, "a detached feed must not reach OnAlarmEvent"); + feed.Disposes.ShouldBe(0, "an injected feed belongs to the injector"); + + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAlarmsAsync(["Tank01.Level.HiHi"], CancellationToken.None); + feed.Emit(); + received.ShouldBe(2, "re-init + re-subscribe must re-wire the feed"); + feed.Starts.ShouldBe(2); + } + + // ───────────────────────────── G-2: one DeployWatcher ───────────────────────────── + + /// + /// Five watchdog cycles (ShutdownAsync → InitializeAsync) leave exactly one open watch + /// stream. Positive control: a real deploy change then raises OnRediscoveryNeeded exactly + /// once — five stacked watchers would raise five times. + /// + [Fact] + public async Task Five_watchdog_cycles_leave_exactly_one_DeployWatcher() + { + var source = new CountingDeployWatchSource(); + var sub = new StreamingSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, + deployWatchSource: source); + var raised = 0; + driver.OnRediscoveryNeeded += (_, _) => Interlocked.Increment(ref raised); + + await driver.InitializeAsync("{}", CancellationToken.None); + for (var cycle = 0; cycle < 5; cycle++) + { + // Let this cycle's watch stream actually open before the next shutdown, so every cycle + // holds a live stream the teardown has to cancel (a loop cancelled before its first + // open would make the count vacuous). + await WaitUntilAsync(() => source.Opened == cycle + 1, $"cycle {cycle}: the watch stream must open"); + await driver.ShutdownAsync(CancellationToken.None); + await driver.InitializeAsync("{}", CancellationToken.None); + } + + await WaitUntilAsync(() => source.Opened == 6 && source.Open == 1, + $"expected six opens and exactly one open watch stream, saw opened={source.Opened} open={source.Open}"); + driver.DeployWatcherForTests.ShouldNotBeNull(); + + // Bootstrap (suppressed) then a genuine change. + await source.PushAsync(1, new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + await source.PushAsync(2, new DateTimeOffset(2026, 1, 2, 0, 0, 0, TimeSpan.Zero)); + await WaitUntilAsync(() => Volatile.Read(ref raised) >= 1, "positive control: a deploy change must raise rediscovery"); + await Task.Delay(200); + Volatile.Read(ref raised).ShouldBe(1, "stacked watchers would each raise for the same deploy"); + + await driver.ShutdownAsync(CancellationToken.None); + await WaitUntilAsync(() => source.Open == 0, "ShutdownAsync must cancel the deploy-watch stream"); + driver.DeployWatcherForTests.ShouldBeNull(); + } + + /// A second InitializeAsync without a shutdown in between replaces the watcher rather than stacking one. + [Fact] + public async Task InitializeAsync_twice_replaces_the_previous_DeployWatcher() + { + var source = new CountingDeployWatchSource(); + var sub = new StreamingSubscriber(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub, + deployWatchSource: source); + + await driver.InitializeAsync("{}", CancellationToken.None); + var first = driver.DeployWatcherForTests; + await WaitUntilAsync(() => source.Opened == 1, "the first watch stream must open"); + await driver.InitializeAsync("{}", CancellationToken.None); + + await WaitUntilAsync(() => source.Opened == 2 && source.Open == 1, + $"expected two opens and one open watch stream, saw opened={source.Opened} open={source.Open}"); + driver.DeployWatcherForTests.ShouldNotBeSameAs(first); + } + + // ───────────────────────────── G-1/G-9: owned runtime, channels + session ───────────────────────────── + + /// + /// The production path (no injected seams): InitializeAsync builds the owned client, session + /// and supervisor; ShutdownAsync closes the session, disposes both channels and nulls the + /// seams; the next InitializeAsync opens a NEW session on NEW channels — the watchdog's + /// recovery now genuinely rebuilds. The session open is replaced by a seam; the gRPC + /// channels are real but lazy (nothing dials until a call is made). + /// + [Fact] + public async Task ShutdownAsync_closes_the_session_disposes_both_channels_and_reinit_rebuilds() + { + var opens = 0; + var source = new CountingDeployWatchSource(); + await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, deployWatchSource: source, + platformSource: new NoPlatforms()) + { + SessionOpenOverrideForTests = _ => { opens++; return Task.CompletedTask; }, + }; + + await driver.InitializeAsync("{}", CancellationToken.None); + opens.ShouldBe(1); + var client = driver.OwnedMxClientForTests.ShouldNotBeNull(); + var session = driver.OwnedMxSessionForTests.ShouldNotBeNull(); + session.IsConnected.ShouldBeTrue(); + driver.SupervisorForTests.ShouldNotBeNull(); + driver.ProbeWatcherForTests.ShouldNotBeNull(); + driver.DeployWatcherForTests.ShouldNotBeNull(); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + var stamp = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); + + await driver.ShutdownAsync(CancellationToken.None); + + session.IsConnected.ShouldBeFalse("ShutdownAsync must close the MX session"); + await Should.ThrowAsync(() => client.OpenSessionAsync(cancellationToken: CancellationToken.None), + "the owned gateway channel must be disposed"); + driver.OwnedMxClientForTests.ShouldBeNull(); + driver.OwnedMxSessionForTests.ShouldBeNull(); + driver.SupervisorForTests.ShouldBeNull(); + driver.ProbeWatcherForTests.ShouldBeNull(); + driver.DeployWatcherForTests.ShouldBeNull(); + await WaitUntilAsync(() => source.Open == 0, "the deploy-watch stream must be cancelled"); + driver.GetHealth().State.ShouldBe(DriverState.Unknown); + driver.GetHealth().LastSuccessfulRead.ShouldBe(stamp); + + // The watchdog's second half: a fresh runtime on the same object. + await driver.InitializeAsync("{}", CancellationToken.None); + opens.ShouldBe(2, "re-init must open a NEW session, not reuse the dead one"); + driver.OwnedMxClientForTests.ShouldNotBeNull().ShouldNotBeSameAs(client); + driver.OwnedMxSessionForTests.ShouldNotBeNull().ShouldNotBeSameAs(session); + driver.OwnedMxSessionForTests!.IsConnected.ShouldBeTrue(); + driver.SupervisorForTests.ShouldNotBeNull(); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + await WaitUntilAsync(() => source.Open == 1, "re-init must start one deploy watcher"); + } + + /// + /// A connect that fails mid-build releases the partial runtime (no orphaned client per + /// watchdog retry), lands Faulted, and the next InitializeAsync starts from nothing. + /// + [Fact] + public async Task Failed_InitializeAsync_releases_the_partial_runtime_and_the_retry_rebuilds() + { + var attempts = 0; + await using var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, + platformSource: new NoPlatforms()) + { + SessionOpenOverrideForTests = _ => ++attempts == 1 + ? throw new InvalidOperationException("gateway refused") + : Task.CompletedTask, + }; + + var ex = await Should.ThrowAsync(() => driver.InitializeAsync("{}", CancellationToken.None)); + ex.Message.ShouldBe("gateway refused"); + driver.OwnedMxClientForTests.ShouldBeNull("a failed connect must not orphan the client"); + driver.OwnedMxSessionForTests.ShouldBeNull(); + driver.GetHealth().State.ShouldBe(DriverState.Faulted); + driver.GetHealth().LastError.ShouldBe("gateway refused"); + + await driver.InitializeAsync("{}", CancellationToken.None); + attempts.ShouldBe(2); + driver.OwnedMxSessionForTests.ShouldNotBeNull().IsConnected.ShouldBeTrue(); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + /// DisposeAsync after ShutdownAsync is harmless, and ShutdownAsync after DisposeAsync is a no-op. + [Fact] + public async Task Shutdown_then_dispose_and_dispose_then_shutdown_are_both_harmless() + { + var sub = new StreamingSubscriber(); + var driver = new GalaxyDriver("drv", Opts(watchDeploy: false), hierarchySource: null, subscriber: sub); + await driver.InitializeAsync("{}", CancellationToken.None); + await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); + + await driver.ShutdownAsync(CancellationToken.None); + await Should.NotThrowAsync(async () => await driver.DisposeAsync()); + await Should.NotThrowAsync(() => driver.ShutdownAsync(CancellationToken.None)); + await Should.ThrowAsync(() => driver.InitializeAsync("{}", CancellationToken.None)); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyMxSessionCloseTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyMxSessionCloseTests.cs new file mode 100644 index 00000000..aa6219c3 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyMxSessionCloseTests.cs @@ -0,0 +1,90 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// bounds the gateway CloseSession round-trip by +/// the caller's token (the SDK dispose closes under its 30 s call timeout) and leaves the session +/// re-openable. +/// +public sealed class GalaxyMxSessionCloseTests +{ + /// A close against a never-answering gateway returns when the token fires, without throwing. + [Fact] + public async Task CloseAsync_returns_at_the_token_deadline_against_a_never_answering_gateway() + { + var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest")) + { + OpenAndRegisterOverrideForTests = _ => Task.CompletedTask, + CloseOverrideForTests = ct => Task.Delay(Timeout.Infinite, ct), + }; + await session.ConnectAsync(null!, CancellationToken.None); + session.IsConnected.ShouldBeTrue(); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + var sw = System.Diagnostics.Stopwatch.StartNew(); + await session.CloseAsync(cts.Token).WaitAsync(TimeSpan.FromSeconds(10)); + sw.Stop(); + + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5)); + session.IsConnected.ShouldBeFalse(); + + // Still usable: a close is not a dispose. + await session.ConnectAsync(null!, CancellationToken.None); + session.IsConnected.ShouldBeTrue(); + } + + /// Positive control: a close that answers completes and reports the token it was handed. + [Fact] + public async Task CloseAsync_completes_when_the_gateway_answers() + { + CancellationToken seen = default; + var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest")) + { + OpenAndRegisterOverrideForTests = _ => Task.CompletedTask, + CloseOverrideForTests = ct => { seen = ct; return Task.CompletedTask; }, + }; + await session.ConnectAsync(null!, CancellationToken.None); + + using var cts = new CancellationTokenSource(); + await session.CloseAsync(cts.Token); + + seen.ShouldBe(cts.Token, "the caller's token must reach the gateway round-trip"); + session.IsConnected.ShouldBeFalse(); + } + + /// + /// The driver's teardown is CloseAsync (bounded) then DisposeAsync (final). The dispose must + /// not issue a second gateway close; a fresh connect after a close must close again (positive + /// control: the guard keys on "was connected", not on "ever closed"). + /// + [Fact] + public async Task CloseAsync_then_DisposeAsync_closes_the_gateway_session_once() + { + var closes = 0; + var session = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest")) + { + OpenAndRegisterOverrideForTests = _ => Task.CompletedTask, + CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; }, + }; + await session.ConnectAsync(null!, CancellationToken.None); + + await session.CloseAsync(CancellationToken.None); + await session.DisposeAsync(); + closes.ShouldBe(1, "dispose after a close must not close the gateway session again"); + + var reopened = new GalaxyMxSession(new GalaxyMxAccessOptions("CloseTest")) + { + OpenAndRegisterOverrideForTests = _ => Task.CompletedTask, + CloseOverrideForTests = _ => { Interlocked.Increment(ref closes); return Task.CompletedTask; }, + }; + await reopened.ConnectAsync(null!, CancellationToken.None); + await reopened.CloseAsync(CancellationToken.None); + await reopened.ConnectAsync(null!, CancellationToken.None); + await reopened.DisposeAsync(); + closes.ShouldBe(3, "each live session closes exactly once"); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs index bfef6df2..6b311f9f 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyTelemetryTests.cs @@ -14,6 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// rather than asserting on internal state, so the /// tests double as documentation of the listener-side contract. /// +[Collection(ProcessWideTelemetryCollection.Name)] public sealed class GalaxyTelemetryTests { /// Subscribes an ActivityListener for the test, captures each spawned activity. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs new file mode 100644 index 00000000..51b4cdfb --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterConcurrencyTests.cs @@ -0,0 +1,170 @@ +using System.Collections.Concurrent; +using Shouldly; +using Xunit; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Review 2026-10-21 G-8: a batch of writes ran strictly serially, three round-trips per +/// first-touch tag. Distinct tags now run as concurrent lanes bounded at +/// ; same-tag writes keep their +/// order inside one lane; results stay at their request's index. +/// +public sealed class GatewayGalaxyDataWriterConcurrencyTests +{ + private const int WaitMs = 10_000; + + private static GatewayGalaxyDataWriter NewWriter() + => new(new GalaxyMxSession(new Config.GalaxyMxAccessOptions(ClientName: "OtOpcUa-Test")), writeUserId: 0); + + /// + /// Fake ops whose AddItem parks every caller until callers are in + /// flight at once (or a safety delay elapses), recording the peak concurrency. + /// + private sealed class GatingOps : IMxWriteOps + { + private int _inFlight; + private int _peak; + private int _nextHandle; + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly ConcurrentDictionary _handles = new(StringComparer.OrdinalIgnoreCase); + + public int ReleaseAt { get; init; } = int.MaxValue; + public int Peak => Volatile.Read(ref _peak); + public ConcurrentQueue<(int Handle, MxValue Value)> RawWrites { get; } = new(); + public ConcurrentQueue AddItems { get; } = new(); + public Func? WriteResponder { get; init; } + + /// Deterministic handle per tag when set; otherwise handles are assigned in AddItem completion order. + public Func? HandleFor { get; init; } + + public string SessionId => "sess"; + public int ServerHandle => 1; + + public async Task AddItemAsync(string fullRef, CancellationToken ct) + { + AddItems.Enqueue(fullRef); + var now = Interlocked.Increment(ref _inFlight); + int seen; + do { seen = Volatile.Read(ref _peak); } while (now > seen && Interlocked.CompareExchange(ref _peak, now, seen) != seen); + if (now >= ReleaseAt) _release.TrySetResult(); + // Park on the signal; the delay is only a safety net against a wedged test (a starved + // pool must not release the gate early and turn the peak assertion flaky). + await Task.WhenAny(_release.Task, Task.Delay(WaitMs, ct)); + Interlocked.Decrement(ref _inFlight); + return _handles.GetOrAdd(fullRef, r => HandleFor?.Invoke(r) ?? Interlocked.Increment(ref _nextHandle)); + } + + public Task InvokeAsync(MxCommandRequest request, CancellationToken ct) + => Task.FromResult(new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } }); + + public Task WriteRawAsync(int itemHandle, MxValue value, int userId, CancellationToken ct) + { + RawWrites.Enqueue((itemHandle, value)); + return Task.FromResult(WriteResponder?.Invoke(itemHandle) ?? GoodStatusRow()); + } + + // A reply carrying a Good MX status ROW (not an empty statuses array): the writer's static + // meters are process-wide, and an empty-statuses Good would increment galaxy.writes.unconfirmed + // under the meter tests running in parallel in another class. + private static MxCommandReply GoodStatusRow() + { + var reply = new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } }; + reply.Statuses.Add(new MxStatusProxy { Success = 1, Category = MxStatusCategory.Ok }); + return reply; + } + } + + private static SecurityClassification Free(string _) => SecurityClassification.FreeAccess; + + /// Sixteen distinct tags: lanes overlap (not serial) and never exceed the cap of eight. + [Fact] + public async Task Distinct_tags_write_concurrently_bounded_at_the_lane_cap() + { + var ops = new GatingOps { ReleaseAt = GatewayGalaxyDataWriter.MaxConcurrentWriteLanes }; + var writes = Enumerable.Range(0, 16).Select(i => new WriteRequest($"Tag{i}.PV", (double)i)).ToArray(); + + var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None) + .WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + ops.Peak.ShouldBe(GatewayGalaxyDataWriter.MaxConcurrentWriteLanes, + "first-touch lanes must overlap (serial = 1) and stay bounded (unbounded > 8)"); + results.Count.ShouldBe(16); + results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good); + ops.AddItems.Count.ShouldBe(16, "one AddItem per distinct tag"); + ops.RawWrites.Count.ShouldBe(16); + } + + /// Writes to the same tag stay in input order inside their lane and share one AddItem. + [Fact] + public async Task Same_tag_writes_stay_ordered_and_share_one_handle() + { + var ops = new GatingOps { ReleaseAt = 2 }; + var writes = new[] + { + new WriteRequest("Tank01.SP", 1.0), + new WriteRequest("Tank02.SP", 10.0), + new WriteRequest("Tank01.SP", 2.0), + new WriteRequest("Tank01.SP", 3.0), + }; + + var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None) + .WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + results.Count.ShouldBe(4); + results.ShouldAllBe(r => r.StatusCode == StatusCodeMap.Good); + ops.AddItems.Count(a => a == "Tank01.SP").ShouldBe(1, "a lane AddItems its tag once"); + var tank01 = ops.RawWrites.Where(w => w.Handle == ops.RawWrites.First(x => x.Value.DoubleValue == 1.0).Handle) + .Select(w => w.Value.DoubleValue).ToArray(); + tank01.ShouldBe([1.0, 2.0, 3.0], "same-tag writes must keep their authored order"); + } + + /// A per-tag failure lands at that request's index while the other lanes stay Good. + [Fact] + public async Task Results_land_at_their_request_index() + { + var ops = new GatingOps + { + ReleaseAt = 3, + HandleFor = tag => tag[0] - 'A' + 1, // A.PV → 1, B.PV → 2, C.PV → 3 + WriteResponder = handle => new MxCommandReply + { + ProtocolStatus = new ProtocolStatus + { + Code = handle == 2 ? ProtocolStatusCode.MxaccessFailure : ProtocolStatusCode.Ok, + }, + }, + }; + var writes = new[] + { + new WriteRequest("A.PV", 1.0), + new WriteRequest("B.PV", 2.0), // refused by the gateway + new WriteRequest("C.PV", 3.0), + }; + + var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None) + .WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + results.Select(r => r.StatusCode).ShouldBe( + [StatusCodeMap.Good, StatusCodeMap.BadCommunicationError, StatusCodeMap.Good], + "each result must sit at its request's index regardless of lane completion order"); + } + + /// A single-tag batch takes the direct path and still produces one result per request. + [Fact] + public async Task Single_tag_batch_runs_serially_in_order() + { + var ops = new GatingOps { ReleaseAt = 1 }; + var writes = new[] { new WriteRequest("X.PV", 1.0), new WriteRequest("X.PV", 2.0) }; + + var results = await NewWriter().WriteBatchForTestAsync(ops, writes, Free, CancellationToken.None) + .WaitAsync(TimeSpan.FromMilliseconds(WaitMs)); + + results.Count.ShouldBe(2); + ops.Peak.ShouldBe(1); + ops.RawWrites.Select(w => w.Value.DoubleValue).ShouldBe([1.0, 2.0]); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs index 67a01d77..83d41dea 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyDataWriterFailClosedTests.cs @@ -13,6 +13,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// of leaving a phantom-Good node. These tests drive the real write pipeline through the internal /// seam (the SDK session types are sealed + internal-ctor and cannot be faked). /// +[Collection(ProcessWideTelemetryCollection.Name)] public sealed class GatewayGalaxyDataWriterFailClosedTests { private static GatewayGalaxyDataWriter NewWriter() diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs new file mode 100644 index 00000000..4513efa6 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorCycleCtsTests.cs @@ -0,0 +1,92 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; + +/// +/// Review 2026-10-21 G-3: allocated a +/// new per recovery cycle and only the last one was ever +/// disposed. Each cycle now releases its own CTS when its loop exits, and +/// awaits the loop instead of blocking on it. +/// +public sealed class ReconnectSupervisorCycleCtsTests +{ + private const int WaitMs = 10_000; + + private static ReconnectOptions FastBackoff() => + new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5), MaxBackoffOverride: TimeSpan.FromMilliseconds(20)); + + /// Three completed cycles ⇒ three released CTSs; the in-flight one is not released early (positive control). + [Fact] + public async Task Every_completed_recovery_cycle_releases_its_own_cts() + { + var holdReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new[] { false }; // read on the recovery loop's thread, written here — Volatile both ways + using var sup = new ReconnectSupervisor( + reopen: async _ => + { + if (Volatile.Read(ref hold[0])) + { + reopenEntered.TrySetResult(); + await holdReopen.Task; + } + }, + replay: _ => Task.CompletedTask, + options: FastBackoff()); + + for (var i = 0; i < 3; i++) + { + sup.ReportTransportFailure(new IOException($"drop {i}")); + await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token); + } + await WaitUntilAsync(() => sup.CyclesReleased == 3, "each completed cycle must release its CTS"); + sup.CyclesStarted.ShouldBe(3); + + // Positive control: a cycle parked inside reopen still holds its CTS. + Volatile.Write(ref hold[0], true); + sup.ReportTransportFailure(new IOException("drop 3")); + (await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task); + sup.CyclesStarted.ShouldBe(4); + sup.CyclesReleased.ShouldBe(3, "an in-flight cycle's CTS is live until its loop exits"); + + holdReopen.TrySetResult(); + await sup.WaitForHealthyAsync(new CancellationTokenSource(WaitMs).Token); + await WaitUntilAsync(() => sup.CyclesReleased == 4, "the fourth cycle must release its CTS on exit"); + } + + /// StopAsync cancels a parked cycle, awaits its exit and releases its CTS without blocking. + [Fact] + public async Task StopAsync_cancels_the_in_flight_cycle_and_releases_its_cts() + { + var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sup = new ReconnectSupervisor( + reopen: async ct => + { + reopenEntered.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + }, + replay: _ => Task.CompletedTask, + options: FastBackoff()); + + sup.ReportTransportFailure(new IOException("drop")); + (await Task.WhenAny(reopenEntered.Task, Task.Delay(WaitMs))).ShouldBe(reopenEntered.Task); + + var stop = sup.StopAsync(); + (await Task.WhenAny(stop, Task.Delay(WaitMs))).ShouldBe(stop, "StopAsync must cancel the parked reopen and return"); + sup.CyclesReleased.ShouldBe(1); + await Should.NotThrowAsync(sup.StopAsync); // idempotent + Should.NotThrow(sup.Dispose); + } + + private static async Task WaitUntilAsync(Func condition, string because) + { + var deadline = DateTime.UtcNow.AddMilliseconds(WaitMs); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new ShouldAssertException(because); + await Task.Delay(10); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs index 21460637..94784a45 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs @@ -31,6 +31,26 @@ public sealed class DiagnoseTests => Diag("""var c = new System.Net.Http.HttpClient(); return 0;""") .ShouldContain(x => x.Severity == 8); + // Sandbox hardening parity (R05:10): the editor runs the SAME ForbiddenTypeAnalyzer the publish gate + // runs, so every newly-denied form must surface as an OTSCRIPT_FORBIDDEN marker here — editor accepts + // ⇔ publish accepts. One case per deny mechanism (namespace, member, type), plus the allow-list. + [Theory] + [InlineData("""new System.Threading.SemaphoreSlim(0).Wait(); return 0;""")] // System.Threading namespace + [InlineData("""System.Threading.SpinWait.SpinUntil(() => false); return 0;""")] + [InlineData("""var t = System.Type.GetType("System.IO.File"); return t is null ? 0 : 1;""")] // forbidden member + [InlineData("""return typeof(string).GetMethods().Length;""")] + [InlineData("""var o = System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(string)); return 0;""")] // forbidden type + public void Hardened_sandbox_forms_are_flagged_in_the_editor(string code) + => Diag(code).ShouldContain(x => x.Code == "OTSCRIPT_FORBIDDEN" && x.Severity == 8); + + [Theory] + [InlineData("""var t = System.Threading.CancellationToken.None; return t.IsCancellationRequested ? 1 : 0;""")] + [InlineData("""var x = 41; System.Threading.Interlocked.Increment(ref x); return x;""")] + [InlineData("""var x = 42; return System.Threading.Volatile.Read(ref x);""")] + [InlineData("""object o = 42; return o.GetType().IsValueType ? 1 : 0;""")] + public void Allow_listed_forms_are_not_flagged_in_the_editor(string code) + => Diag(code).ShouldBeEmpty(); + [Fact] public void Dynamic_tag_path_is_flagged() => Diag("""var p = "A"; return ctx.GetTag(p).Value;""") .ShouldContain(x => x.Message.Contains("literal")); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs new file mode 100644 index 00000000..95a2b0fc --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs @@ -0,0 +1,159 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:3.1 — the admin singleton's mailbox used to be suspended for the whole of every +/// ReceiveAsync: one Test Connect against a black-holed PLC (60 s clamp) or one large deploy +/// compose stalled every other admin operation fleet-wide, alarm acks included. Both now run off the +/// actor thread; only the "is one in flight" check stays serialised. +/// +public sealed class AdminOperationsActorMailboxTests : ControlPlaneActorTestBase +{ + private static readonly TimeSpan AckBudget = TimeSpan.FromSeconds(1); + + /// An alarm ack completes in under a second while a 60 s Test Connect is still pending. + [Fact] + public void Alarm_ack_completes_while_a_60s_test_connect_is_pending() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe("coord"); + var probe = new BlockingProbe(); + var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe })); + var connectClient = CreateTestProbe("connect-client"); + + connectClient.Send(actor, new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 60, Guid.NewGuid())); + probe.Started.Wait(PresenceBudget).ShouldBeTrue("the probe must be in flight before the ack is sent"); + + actor.Tell(new AcknowledgeAlarmCommand("alarm-1", "op", null, CorrelationId.NewId())); + var ack = ExpectMsg(AckBudget); + ack.Ok.ShouldBeTrue(); + + // Positive control: the probe reply still lands, with the right sender, once the probe returns. + connectClient.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + probe.Release(new DriverProbeResult(true, null, TimeSpan.FromMilliseconds(5))); + var result = connectClient.ExpectMsg(PresenceBudget); + result.Ok.ShouldBeTrue(); + result.LatencyMs.ShouldNotBeNull(); + } + + /// A probe that times out still answers its own sender with the clamped timeout message. + [Fact] + public void Test_connect_timeout_is_reported_to_its_sender() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe("coord"); + var probe = new BlockingProbe(); // never released: the CTS fires at the 1 s clamp floor + var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe })); + + actor.Tell(new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 1, Guid.NewGuid())); + + var result = ExpectMsg(PresenceBudget); + result.Ok.ShouldBeFalse(); + result.Message.ShouldNotBeNull(); + result.Message.ShouldContain("timed out"); + } + + /// + /// The deploy compose (snapshot + validate + gzip + row write) runs off the actor: an alarm ack + /// completes while the compose is parked on a gated DB context, and the deploy still completes + /// (Accepted + dispatched) once the gate opens. + /// + [Fact] + public void Alarm_ack_completes_while_a_deploy_compose_is_in_flight() + { + var gated = new GatedDbFactory(NewInMemoryDbFactory()); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty())); + var deployer = CreateTestProbe("deployer"); + + gated.Hold(); + deployer.Send(actor, new StartDeployment("joe", CorrelationId.NewId())); + gated.Entered.Wait(PresenceBudget).ShouldBeTrue("the compose must be parked before the ack is sent"); + + actor.Tell(new AcknowledgeAlarmCommand("alarm-2", "op", null, CorrelationId.NewId())); + ExpectMsg(AckBudget).Ok.ShouldBeTrue(); + + // Positive control: the deploy completes once released. + deployer.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + gated.Release(); + deployer.ExpectMsg(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.ExpectMsg(PresenceBudget); + } + + /// + /// Only the in-flight check is serialised — and it must still catch a second request that arrives + /// while the first compose has not yet written its row (the DB alone would let both through). + /// + [Fact] + public void Second_StartDeployment_during_an_in_flight_compose_is_refused() + { + var gated = new GatedDbFactory(NewInMemoryDbFactory()); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty())); + var first = CreateTestProbe("first"); + var second = CreateTestProbe("second"); + + gated.Hold(); + first.Send(actor, new StartDeployment("joe", CorrelationId.NewId())); + gated.Entered.Wait(PresenceBudget).ShouldBeTrue(); + + second.Send(actor, new StartDeployment("kim", CorrelationId.NewId())); + second.ExpectMsg(AckBudget).Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight); + + gated.Release(); + first.ExpectMsg(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.ExpectMsg(PresenceBudget); + } + + private sealed class BlockingProbe : IDriverProbe + { + private readonly TaskCompletionSource _tcs = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ManualResetEventSlim Started { get; } = new(); + public string DriverType => "Blocking"; + + public async Task ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct) + { + Started.Set(); + return await _tcs.Task.WaitAsync(ct); + } + + public void Release(DriverProbeResult result) => _tcs.TrySetResult(result); + } + + /// + /// Passes synchronous context creation through (the serialised in-flight check) and parks the + /// asynchronous one (the off-actor compose) on a gate while is in effect. + /// + private sealed class GatedDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private volatile TaskCompletionSource? _gate; + + public ManualResetEventSlim Entered { get; } = new(); + + public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _gate?.TrySetResult(); + + public OtOpcUaConfigDbContext CreateDbContext() => inner.CreateDbContext(); + + public async Task CreateDbContextAsync(CancellationToken cancellationToken = default) + { + Entered.Set(); + if (_gate is { } gate) await gate.Task.WaitAsync(cancellationToken); + return inner.CreateDbContext(); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs new file mode 100644 index 00000000..f4e6fd22 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs @@ -0,0 +1,68 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:4.1 — admin operations echoed ex.Message to the UI, so a SqlException's connection-string +/// fragment reached any ConfigEditor browser session. The reply is now a fixed sentence plus the request's +/// correlation id; the detail stays in the central log. +/// +public sealed class AdminOperationsActorServerFaultMessageTests : ControlPlaneActorTestBase +{ + private const string Secret = "Server=sql-prod-01;Password=hunter2"; + + [Fact] + public void StartDeployment_server_fault_reply_carries_the_correlation_id_and_not_the_exception_text() + { + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(new ThrowingDbFactory(), coordinator.Ref, Enumerable.Empty())); + var correlationId = CorrelationId.NewId(); + + actor.Tell(new StartDeployment("joe", correlationId)); + + var reply = ExpectMsg(PresenceBudget); + reply.Outcome.ShouldBe(StartDeploymentOutcome.Rejected); + reply.Message.ShouldNotBeNull(); + reply.Message.ShouldNotContain("hunter2"); + reply.Message.ShouldNotContain("sql-prod-01"); + reply.Message.ShouldContain(correlationId.ToString()); // the operator can find the log line + } + + [Fact] + public void TestDriverConnect_probe_throw_reply_carries_the_correlation_id_and_not_the_exception_text() + { + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(NewInMemoryDbFactory(), coordinator.Ref, new IDriverProbe[] { new ThrowingProbe() })); + var correlationId = Guid.NewGuid(); + + actor.Tell(new TestDriverConnect("Throwing", "{}", 5, correlationId)); + + var reply = ExpectMsg(PresenceBudget); + reply.Ok.ShouldBeFalse(); + reply.Message.ShouldNotBeNull(); + reply.Message.ShouldNotContain("hunter2"); + reply.Message.ShouldContain(correlationId.ToString()); + } + + private sealed class ThrowingDbFactory : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => + throw new InvalidOperationException("Cannot open database: " + Secret); + } + + private sealed class ThrowingProbe : IDriverProbe + { + public string DriverType => "Throwing"; + public Task ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct) => + throw new InvalidOperationException("probe blew up: " + Secret); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs new file mode 100644 index 00000000..aa5ded29 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs @@ -0,0 +1,133 @@ +using System.IO.Compression; +using System.Text; +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Storage; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:3.2 — StartDeployment used to read the whole config DB twice: once for the validator, +/// once for the composer. An edit landing between the two shipped an artifact the validator never saw. +/// The deploy now validates and composes ONE snapshot, so an edit that lands mid-deploy is either +/// validated (it was in the snapshot) or absent from the artifact (it was not) — never shipped unseen. +/// +public sealed class AdminOperationsActorSingleSnapshotTests : ControlPlaneActorTestBase +{ + private const string LateEquipmentId = "EQ-late-and-NOT-canonical"; + + /// + /// An Equipment with a non-canonical id (the validator rejects those with EquipmentIdNotDerived) + /// is committed by another context while the snapshot read is in progress, AFTER the Equipment table + /// has been read. The validator cannot see it, so the deploy is Accepted — and the artifact must not + /// carry it either. Positive control: the row really landed in the DB, and a deploy started after it + /// is Rejected on exactly that rule. + /// + [Fact] + public void Edit_landing_between_validate_and_compose_cannot_ship_unvalidated() + { + // One shared root: an InMemory store is scoped to the internal service provider, and a context + // carrying an interceptor builds its own provider — without the shared root the intercepting + // context would read an EMPTY database and the whole test would be vacuous. + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var plain = new RootedDbFactory(dbName, root, interceptor: null); + using (var db = plain.CreateDbContext()) + { + // A Tag to materialise — Tags is read AFTER Equipment by the snapshot factory, so an insert + // fired on the first Tag lands "between" the two tables of one read. + db.DriverInstances.Add(new DriverInstance + { + DriverInstanceId = "d", ClusterId = "", Name = "drv", DriverType = "Modbus", DriverConfig = "{}", + }); + db.Devices.Add(new Device { DeviceId = "dev", DriverInstanceId = "d", Name = "dev", DeviceConfig = "{}" }); + db.Tags.Add(new Tag + { + TagId = "tag-speed", DeviceId = "dev", Name = "speed", DataType = "Float", + AccessLevel = TagAccessLevel.Read, TagConfig = "{}", + }); + db.SaveChanges(); + } + + var injecting = new RootedDbFactory(dbName, root, new LateEquipmentInterceptor(plain)); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(injecting, coordinator.Ref, Enumerable.Empty())); + + actor.Tell(new StartDeployment("joe", CorrelationId.NewId())); + + var reply = ExpectMsg(PresenceBudget); + reply.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, reply.Message); + coordinator.ExpectMsg(PresenceBudget); + + using (var verify = plain.CreateDbContext()) + { + // The injection happened... + verify.Equipment.Any(e => e.EquipmentId == LateEquipmentId).ShouldBeTrue(); + // ...and the shipped artifact does not contain what the validator never saw. + var row = verify.Deployments.Single(); + Inflate(row.ArtifactBlob).ShouldNotContain(LateEquipmentId); + // The coordinator is a probe here, so seal the row by hand to let the next deploy through. + row.Status = DeploymentStatus.Sealed; + verify.SaveChanges(); + } + + // Positive control: now that the row is visible to a fresh read, the gate rejects it. + actor.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var second = ExpectMsg(PresenceBudget); + second.Outcome.ShouldBe(StartDeploymentOutcome.Rejected); + second.Message.ShouldNotBeNull(); + second.Message.ShouldContain("EquipmentIdNotDerived"); + } + + private static string Inflate(byte[] blob) + { + using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress); + using var reader = new StreamReader(gz, Encoding.UTF8); + return reader.ReadToEnd(); + } + + /// On the first Tag materialised, commits a non-canonical Equipment through ANOTHER context. + private sealed class LateEquipmentInterceptor(IDbContextFactory other) + : IMaterializationInterceptor + { + private int _fired; + + public object InitializedInstance(MaterializationInterceptionData materializationData, object entity) + { + if (entity is Tag && Interlocked.Exchange(ref _fired, 1) == 0) + { + using var db = other.CreateDbContext(); + db.Equipment.Add(new Equipment + { + EquipmentUuid = Guid.NewGuid(), EquipmentId = LateEquipmentId, Name = "late", + UnsLineId = "line-a", MachineCode = "m", + }); + db.SaveChanges(); + } + return entity; + } + } + + private sealed class RootedDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor? interceptor) + : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() + { + var builder = new DbContextOptionsBuilder().UseInMemoryDatabase(dbName, root); + if (interceptor is not null) builder.AddInterceptors(interceptor); + return new OtOpcUaConfigDbContext(builder.Options); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs new file mode 100644 index 00000000..aba37648 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs @@ -0,0 +1,191 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.EntityFrameworkCore.Update; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:2.6 — one constraint hit used to drop the whole flush. The writer now detaches the rejected +/// row(s) and re-saves, so a duplicate EventId costs exactly that row. +/// +public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase +{ + [Fact] + public void A_rejected_row_is_dropped_and_the_rest_of_the_batch_is_persisted() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var duplicate = Guid.NewGuid(); + var rejecting = new RejectingDbFactory(dbName, root, new AlwaysRejectBatchContaining(duplicate)); + var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + + Watch(actor); + actor.Tell(PoisonPill.Instance); // PostStop flushes + ExpectTerminated(actor); + + using var db = rejecting.CreateDbContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(20, "every row but the rejected one lands"); + persisted.ShouldNotContain(duplicate); + // Positive control: the rows on either side of the rejected one are present. + persisted.ShouldContain(ids[6]); + persisted.ShouldContain(ids[8]); + } + + /// + /// Fix round 1 / I4 — a FAITHFUL SQL Server shape: the provider sends the batch as one statement batch, + /// so the for one duplicate names EVERY entry in it. Detaching + /// ex.Entries therefore dropped the whole flush. The writer must drop only the rows whose + /// EventId is already persisted and keep the rest. + /// + [Fact] + public void A_whole_batch_rejection_drops_only_the_rows_already_persisted() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var factory = new RejectingDbFactory(dbName, root, new RejectWholeBatchOnDuplicate()); + var duplicate = Guid.NewGuid(); + using (var seed = factory.CreateDbContext()) + { + seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate }); + seed.SaveChanges(); + } + var actor = Sys.ActorOf(AuditWriterActor.Props(factory)); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor); + + using var db = factory.CreateDbContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(21, "the 20 new rows land beside the one that was already there"); + persisted.Count(id => id == duplicate).ShouldBe(1); + db.ConfigAuditLogs.Single(r => r.EventId == duplicate).Principal.ShouldBe("earlier"); + } + + /// The same contract against the REAL provider and the real filtered unique index. + [Fact] + [Trait("Category", "SchemaCompliance")] + public void On_sql_server_a_duplicate_event_id_costs_only_that_row() + { + using var sql = new SqlServerTestDatabase(); + var duplicate = Guid.NewGuid(); + using (var seed = sql.NewContext()) + { + seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate }); + seed.SaveChanges(); + } + var actor = Sys.ActorOf(AuditWriterActor.Props(sql.Factory())); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor, PresenceBudget); + + using var db = sql.NewContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(21); + persisted.Count(id => id == duplicate).ShouldBe(1); + foreach (var id in ids) persisted.ShouldContain(id); + } + + /// A failure that names no entry is a whole-batch failure, exactly as before. + [Fact] + public void A_connection_level_failure_still_drops_the_batch_without_looping() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var rejecting = new RejectingDbFactory(dbName, root, new AlwaysFailWithoutEntries()); + var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); + + for (var i = 0; i < 5; i++) actor.Tell(NewEvent(Guid.NewGuid())); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor); // terminates ⇒ the retry loop did not spin + + using var db = rejecting.CreateDbContext(); + db.ConfigAuditLogs.Count().ShouldBe(0); + } + + private static AuditEvent NewEvent(Guid eventId) => new() + { + EventId = eventId, Category = "Config", Action = "Edit", Actor = "joe", + OccurredAtUtc = DateTimeOffset.UtcNow, Outcome = AuditOutcome.Success, + DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(), + }; + + /// + /// A row the database refuses on its own terms (not a duplicate — e.g. a per-row constraint), every + /// time it is in the save. Faithful to SQL Server's batch shape: the exception names EVERY added + /// entry, so only a row-by-row retry can isolate the offender. + /// + private sealed class AlwaysRejectBatchContaining(Guid eventId) : ISaveChangesInterceptor + { + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) + { + if (eventData.Context is null) return result; + var added = eventData.Context.ChangeTracker.Entries() + .Where(e => e.State == EntityState.Added).ToList(); + if (!added.Any(e => e.Entity.EventId == eventId)) return result; +#pragma warning disable EF1001 + var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList(); +#pragma warning restore EF1001 + throw new DbUpdateException("The statement conflicted with a CHECK constraint (simulated, whole batch)", entries); + } + } + + /// + /// Models SQL Server's batched insert: when ANY added row's EventId is already stored, the whole save + /// fails with an exception naming EVERY added entry (the InMemory provider enforces no unique index, + /// so this interceptor is the constraint). + /// + private sealed class RejectWholeBatchOnDuplicate : ISaveChangesInterceptor + { + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) + { + if (eventData.Context is not OtOpcUaConfigDbContext ctx) return result; + var added = ctx.ChangeTracker.Entries().Where(e => e.State == EntityState.Added).ToList(); + var ids = added.Select(e => e.Entity.EventId).Where(id => id != null).ToList(); + if (ids.Count == 0 || !ctx.ConfigAuditLogs.AsNoTracking().Any(r => ids.Contains(r.EventId))) return result; +#pragma warning disable EF1001 + var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList(); +#pragma warning restore EF1001 + throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601, whole batch)", entries); + } + } + + private sealed class AlwaysFailWithoutEntries : ISaveChangesInterceptor + { + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) => + throw new DbUpdateException("connection lost (simulated)"); + } + + private sealed class RejectingDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor interceptor) + : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => new( + new DbContextOptionsBuilder() + .UseInMemoryDatabase(dbName, root) + .AddInterceptors(interceptor) + .Options); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs index 9843c8fe..51dcd046 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs @@ -930,6 +930,34 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase /// A spawned gRPC-mode actor plus the two seams the tests observe it through. private sealed record GrpcRig(IActorRef Actor, RecordingDialer Dialer, RecordingPskProvider Psk); + /// + /// R03:1.3 — every contact refresh hands the dialer the CURRENT endpoint set so it can release the + /// channels of endpoints that left it. + /// + [Fact] + public void Each_contact_refresh_tells_the_dialer_which_endpoints_remain() + { + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false), ("host-b:4053", "host-b", 4056, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + // The dialer is resolved lazily on first use; a dispatch forces that, and the NEXT refresh prunes. + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5)); + + using (var db = factory.CreateDbContext()) + { + db.ClusterNodes.Single(n => n.NodeId == "host-b:4053").MaintenanceMode = true; + db.SaveChanges(); + } + rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts()); + + AwaitCondition( + () => rig.Dialer.RetainCalls.Any(set => set.Count == 1 && set.Single().Contains("host-a")), + TimeSpan.FromSeconds(5)); + rig.Dialer.RetainCalls.Last().ShouldBe(new[] { "http://host-a:4056" }); + } + private GrpcRig SpawnGrpcRig( IDbContextFactory factory, RecordingDialer? dialer = null, @@ -1014,6 +1042,19 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase : Task.CompletedTask; } + /// Every endpoint set handed to , in order. + public IReadOnlyList> RetainCalls + { + get { lock (_gate) return _retainCalls.ToList(); } + } + + private readonly List> _retainCalls = []; + + public void RetainOnly(IReadOnlyCollection nodeEndpoints) + { + lock (_gate) _retainCalls.Add(nodeEndpoints.ToList()); + } + /// One recorded dial. public sealed record Dial(string Endpoint, CommandEnvelope Envelope, string ClusterId); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs index 2d110eb6..fc177361 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs @@ -224,6 +224,35 @@ public sealed class MeshGrpcCommandDialerTests Should.NotThrow(dialer.Dispose); } + /// + /// R03:1.3 — a channel for an endpoint that left the dispatch set (host renamed, port moved) is + /// released on the next refresh instead of living, with its idle connection, for the life of central. + /// Positive control: a dial to that endpoint afterwards rebuilds a channel. + /// + [Fact] + public async Task RetainOnly_releases_channels_for_endpoints_that_left_the_dispatch_set() + { + var dialer = new MeshGrpcCommandDialer( + new FakePskProvider(("SITE-A", "psk-site-a")), + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }), + NullLogger.Instance); + foreach (var endpoint in new[] { "http://127.0.0.1:1", "http://127.0.0.1:2" }) + { + await Should.ThrowAsync( + () => dialer.DispatchAsync(endpoint, Envelope(), "SITE-A", CancellationToken.None)); + } + dialer.CachedChannelCount.ShouldBe(2); + + dialer.RetainOnly(new[] { "http://127.0.0.1:1" }); + + dialer.CachedChannelCount.ShouldBe(1); + + await Should.ThrowAsync( + () => dialer.DispatchAsync("http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None)); + dialer.CachedChannelCount.ShouldBe(2, "a returning endpoint gets a fresh channel"); + Should.NotThrow(dialer.Dispose); + } + [Fact] public void It_activates_from_the_container_despite_the_optional_client_factory_parameter() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs new file mode 100644 index 00000000..34129235 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs @@ -0,0 +1,200 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; +using ZB.MOM.WW.Secrets.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; + +/// +/// R03:3.4 — central invalidated every cached key on every contact refresh (default 60 s), so the +/// first inbound ack per cluster per minute blocked a gRPC thread on a synchronous, 3 s-bounded store +/// read and was denied past the bound. pre-resolves the +/// keys off the request path and swaps them in atomically. +/// +public sealed class MeshPskProviderRefreshTests +{ + private static MeshPskProvider Provider(ISecretResolver resolver) => + new(Options.Create(new MeshTransportOptions()), resolver, NullLogger.Instance); + + [Fact] + public async Task After_a_refresh_the_lookup_is_served_without_touching_the_store() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + resolver.Calls.ShouldBe(1); + + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-1"); + resolver.Calls.ShouldBe(1, "the request path did not consult the store"); + provider.Matches("SITE-A", "k-1").ShouldBeTrue(); + } + + /// + /// While a refresh is in flight the PREVIOUS key is still served (no invalidate-then-reread window); + /// once it completes the rotated key is served. Positive control on both sides of the swap. + /// + [Fact] + public async Task A_rotated_key_is_swapped_in_atomically_with_the_old_key_served_until_then() + { + var resolver = new GatedResolver { Value = "old" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + provider.TryGetPsk("SITE-A", out var before).ShouldBeTrue(); + before.ShouldBe("old"); + + resolver.Value = "new"; + resolver.Hold(); + var refresh = provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + resolver.Entered.Wait(TimeSpan.FromSeconds(5)).ShouldBeTrue(); + + // Mid-refresh: the old key, from memory — the store is NOT re-read on this path. + var callsMidRefresh = resolver.Calls; + provider.TryGetPsk("SITE-A", out var during).ShouldBeTrue(); + during.ShouldBe("old"); + resolver.Calls.ShouldBe(callsMidRefresh); + + resolver.Release(); + await refresh; + provider.TryGetPsk("SITE-A", out var after).ShouldBeTrue(); + after.ShouldBe("new"); + } + + [Fact] + public async Task A_cluster_whose_refresh_fails_keeps_its_previous_key() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + resolver.Throw = new InvalidOperationException("store offline"); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-1"); + } + + /// + /// Fix round 1 / I8 — "the store says there is no such secret" is a REVOCATION, not a read failure: + /// the previous key must stop being served (from the refreshed snapshot AND the lazy cache). A thrown + /// read keeps the previous key (the test above); a definitive not-found drops it. + /// + [Fact] + public async Task A_secret_deleted_from_the_store_revokes_the_key_on_the_next_refresh() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + provider.Matches("SITE-A", "k-1").ShouldBeTrue("positive control: the key was in service"); + + resolver.Value = null; // the secret was deleted + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); + provider.Matches("SITE-A", "k-1").ShouldBeFalse(); + } + + /// The same revocation reaches a key that was first served from the LAZY cache, not a refresh. + [Fact] + public async Task A_deleted_secret_also_evicts_a_lazily_cached_key() + { + var resolver = new GatedResolver { Value = "k-lazy" }; + var provider = Provider(resolver); + provider.TryGetPsk("SITE-A", out var lazy).ShouldBeTrue(); + lazy.ShouldBe("k-lazy"); + + resolver.Value = null; + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); + } + + /// Fix round 1 minor — the refresh fans out to the store with BOUNDED concurrency, not one read per cluster at once. + [Fact] + public async Task Refresh_store_reads_are_concurrency_bounded() + { + var resolver = new ConcurrencyTrackingResolver(); + var provider = Provider(resolver); + var clusters = Enumerable.Range(0, 40).Select(i => $"SITE-{i}").ToList(); + + await provider.RefreshAsync(clusters, CancellationToken.None); + + resolver.Calls.ShouldBe(40); + resolver.MaxConcurrent.ShouldBeLessThanOrEqualTo(MeshPskProvider.MaxConcurrentRefreshReads); + resolver.MaxConcurrent.ShouldBeGreaterThan(1, "still concurrent, just bounded"); + foreach (var c in clusters) provider.TryGetPsk(c, out _).ShouldBeTrue(); + } + + private sealed class ConcurrencyTrackingResolver : ISecretResolver + { + private int _inFlight; + private int _max; + private int _calls; + public int MaxConcurrent => Volatile.Read(ref _max); + public int Calls => Volatile.Read(ref _calls); + + public async Task GetAsync(SecretName name, CancellationToken ct) + { + Interlocked.Increment(ref _calls); + var now = Interlocked.Increment(ref _inFlight); + int seen; + while (now > (seen = Volatile.Read(ref _max)) && Interlocked.CompareExchange(ref _max, now, seen) != seen) { } + await Task.Delay(20, ct); + Interlocked.Decrement(ref _inFlight); + return "k-" + name.Value; + } + } + + /// A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour). + [Fact] + public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily() + { + var resolver = new GatedResolver { Value = "k-b" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-B", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-b"); + resolver.Calls.ShouldBe(2); + } + + [Fact] + public async Task InvalidateAll_drops_refreshed_keys_too() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.InvalidateAll(); + resolver.Value = "k-2"; + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-2"); + } + + private sealed class GatedResolver : ISecretResolver + { + private int _calls; + private volatile TaskCompletionSource? _gate; + + public string? Value { get; set; } + public Exception? Throw { get; set; } + public int Calls => Volatile.Read(ref _calls); + public ManualResetEventSlim Entered { get; } = new(); + + public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _gate?.TrySetResult(); + + public async Task GetAsync(SecretName name, CancellationToken ct) + { + Interlocked.Increment(ref _calls); + Entered.Set(); + if (_gate is { } gate) await gate.Task.WaitAsync(ct); + if (Throw is not null) throw Throw; + return Value; + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs new file mode 100644 index 00000000..67ac26bd --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs @@ -0,0 +1,87 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Fix round 1 minor — revision-hash stability across the R03:3.2 refactor. The pre-change composer read +/// every table with a SQL ORDER BY (the database's case-INSENSITIVE collation); the refactor re-sorted +/// in memory with an ORDINAL comparer, which orders mixed-case keys differently ("B-1" before "a-2"). The +/// same configuration would then hash differently after the upgrade: drift detection reports every node +/// drifted, and a re-deploy of an unchanged config shows a "new" revision. This pins the composer's output +/// BYTE-identical to a verbatim replica of the pre-change composer on the real provider. +/// +[Trait("Category", "SchemaCompliance")] +public sealed class ConfigComposerSqlServerHashStabilityTests +{ + [Fact] + public async Task Composer_output_is_byte_identical_to_the_pre_change_composer_for_mixed_case_keys() + { + using var sql = new SqlServerTestDatabase(); + await using (var seed = sql.NewContext()) + { + // Collation order a-2, B-1, c-3; ordinal order B-1, a-2, c-3. + foreach (var id in new[] { "c-3", "B-1", "a-2" }) + seed.Scripts.Add(new Script { ScriptId = id, Name = id, SourceCode = "return 1;", SourceHash = id }); + await seed.SaveChangesAsync(); + } + + await using var legacyDb = sql.NewContext(); + var legacy = await LegacySnapshotAndFlattenAsync(legacyDb); + + await using var viaDb = sql.NewContext(); + var current = await ConfigComposer.SnapshotAndFlattenAsync(viaDb); + current.Blob.ShouldBe(legacy.Blob); + current.RevisionHash.ShouldBe(legacy.RevisionHash); + + // The deploy path's form (validate + compose from ONE snapshot) must agree too. + await using var deployDb = sql.NewContext(); + var viaSnapshot = ConfigComposer.Compose(await DraftSnapshotFactory.FromConfigDbAsync(deployDb)); + viaSnapshot.RevisionHash.ShouldBe(legacy.RevisionHash); + } + + /// Verbatim replica of ConfigComposer.SnapshotAndFlattenAsync before R03:3.2 (ad2b5c40). + private static async Task<(byte[] Blob, string RevisionHash)> LegacySnapshotAndFlattenAsync(OtOpcUaConfigDbContext db) + { + var ct = CancellationToken.None; + var snapshot = new + { + Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), + Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), + RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), + DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), + Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), + TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), + Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), + Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), + UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), + PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), + UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), + UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), + NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), + Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), + VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), + ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + }; + var json = JsonSerializer.SerializeToUtf8Bytes(snapshot, new JsonSerializerOptions + { + WriteIndented = false, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never, + }); + using var output = new MemoryStream(json.Length / 8); + using (var gzip = new GZipStream(output, CompressionLevel.Fastest, leaveOpen: true)) + gzip.Write(json, 0, json.Length); + var blob = output.ToArray(); + return (blob, Convert.ToHexStringLower(SHA256.HashData(blob))); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs index 4e431151..674ececd 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs @@ -222,6 +222,47 @@ public sealed class ConfigComposerTests : ControlPlaneActorTestBase withoutOverride.ResilienceConfig.ShouldBeNull("an unset override stays null ⇒ tier defaults at spawn"); } + /// + /// R03:3.2 — the deploy path composes from the it + /// validated rather than re-reading the DB. over the factory's + /// snapshot must therefore be byte-identical to the DB overload, or the drift probe (which still + /// uses the DB overload) would report every deployment as drifted. + /// + [Fact] + public async Task Compose_over_the_factory_snapshot_is_byte_identical_to_the_db_overload() + { + var f = NewInMemoryDbFactory(); + await using (var db = f.CreateDbContext()) + { + db.ServerClusters.Add(NewCluster("cluster-b")); + db.ServerClusters.Add(NewCluster("cluster-a")); + db.ApiKeys.Add(new ApiKey + { + ApiKeyId = "k1", Name = "scada", SecretHash = "h", Principal = "scadabridge", CreatedUtc = FixedTimestamp, + }); + db.Scripts.Add(new Script { ScriptId = "s-1", Name = "s", SourceCode = "return 1;", SourceHash = "x" }); + await db.SaveChangesAsync(); + } + + var viaDb = await ConfigComposer.SnapshotAndFlattenAsync(f.CreateDbContext()); + var snapshot = await Configuration.Validation.DraftSnapshotFactory.FromConfigDbAsync(f.CreateDbContext()); + var viaSnapshot = ConfigComposer.Compose(snapshot); + + viaSnapshot.RevisionHash.ShouldBe(viaDb.RevisionHash); + viaSnapshot.Blob.ShouldBe(viaDb.Blob); + // The composed content is the snapshot's, not something re-read: the keys and clusters are in it. + var json = Inflate(viaSnapshot.Blob); + json.ShouldContain("\"ApiKeyId\":\"k1\""); + json.IndexOf("cluster-a", StringComparison.Ordinal).ShouldBeLessThan(json.IndexOf("cluster-b", StringComparison.Ordinal)); + } + + private static string Inflate(byte[] blob) + { + using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress); + using var reader = new StreamReader(gz, Encoding.UTF8); + return reader.ReadToEnd(); + } + private static readonly DateTime FixedTimestamp = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); private static ServerCluster NewCluster(string id) => new() diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs new file mode 100644 index 00000000..8c93fb1c --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -0,0 +1,440 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Review 03 §2.2: a Deployment row left in Dispatching with NO NodeDeploymentState +/// rows — HandleDispatch threw before its SaveChanges, or the process died between the +/// AdminOps row write and the coordinator's Tell — used to be recovered with an EMPTY expected +/// set, at which point the deadline hit the 0 == 0 "race" branch and returned without touching +/// the row. Every later StartDeployment was then refused AnotherDeploymentInFlight forever. +/// +public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTestBase +{ + private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64)); + private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5); + + /// + /// A lost dispatch that is still inside the apply deadline is re-driven: PreStart re-dispatches it, + /// the expected set is seeded from the enabled rows, and the node's ack seals it. + /// + [Fact] + public void Recovered_row_with_no_expected_set_is_redispatched_and_then_seals() + { + var dbFactory = NewInMemoryDbFactory(); + var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + SeedNodes(dbFactory, "site-a-1:4053"); + + // No DispatchDeployment is ever sent — PreStart alone has to recover it. + var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" }); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// + /// A lost dispatch that is already past the apply deadline is closed as TimedOut with a reason + /// naming the condition — never left in Dispatching — and the next deployment proceeds + /// (positive control: it is dispatched, acked and sealed through the same coordinator). + /// + [Fact] + public void Recovered_row_past_deadline_with_no_expected_set_is_timed_out_not_wedged() + { + var dbFactory = NewInMemoryDbFactory(); + var staleId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); + SeedNodes(dbFactory, "site-a-1:4053"); + + var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + var row = db.Deployments.Single(d => d.DeploymentId == staleId.Value); + row.Status.ShouldBe(DeploymentStatus.TimedOut); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("lost"); + }, duration: PresenceBudget); + + // Positive control: the coordinator is not wedged — a fresh deployment runs end to end. + var nextId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + actor.Tell(new DispatchDeployment(nextId, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + actor.Tell(new ApplyAck(nextId, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// + /// An AwaitingApplyAcks row with no rows (the process died between the status flip and the + /// empty-fleet seal) is re-driven too, and with no enabled nodes it seals empty exactly as a fresh + /// dispatch would — the recovery keys on the missing expected set, not on the status value. + /// + [Fact] + public void Recovered_AwaitingApplyAcks_row_with_no_rows_and_no_fleet_seals_empty() + { + var dbFactory = NewInMemoryDbFactory(); + SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow); + Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + db.NodeDeploymentStates.ShouldBeEmpty(); + }, duration: PresenceBudget); + } + + /// + /// The brief's scenario end to end: HandleDispatch throws once (a transient fault seeding the + /// expected set). The coordinator restarts, recovers the Dispatching row, re-dispatches it and + /// the deployment completes; a second StartDeployment is then Accepted (positive control: a + /// normal deploy seals). + /// + [Fact] + public async Task HandleDispatch_throw_is_recovered_and_the_next_StartDeployment_succeeds() + { + var dbName = Guid.NewGuid().ToString("N"); + var adminFactory = NewInMemoryDbFactory(dbName); + var faulting = new OneShotFaultingDbFactory(NewInMemoryDbFactory(dbName)); + SeedNodes(adminFactory, "site-a-1:4053"); + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout)); + var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty())); + + // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and + // the actor would be stopped, not restarted). Identify is processed after creation, so a + // resolved selection proves PreStart completed. + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); + faulting.FailNext(); + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var first = ExpectMsg(PresenceBudget); + first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + first.DeploymentId.ShouldNotBeNull(); + + // The fault fired, the actor restarted, PreStart re-dispatched: the expected set lands. + AwaitAssert(() => + { + faulting.Faulted.ShouldBeTrue(); + using var db = adminFactory.CreateDbContext(); + db.NodeDeploymentStates.Count(s => s.DeploymentId == first.DeploymentId.Value.Value).ShouldBe(1); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + coordinator.Tell(new ApplyAck(first.DeploymentId.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = adminFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + + // Positive control: nothing is wedged — the next deploy is Accepted and seals too. + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var second = ExpectMsg(PresenceBudget); + second.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.Tell(new ApplyAck(second.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = adminFactory.CreateDbContext(); + db.Deployments.Count(d => d.Status == DeploymentStatus.Sealed).ShouldBe(2); + }, duration: PresenceBudget); + } + + /// + /// Fix round 1 / I7 — the window PreStart cannot see: the Dispatching row is committed AFTER the + /// coordinator's PreStart ran, and the coordinator is never told (the admin singleton died between + /// its commit and the Tell). Nothing would ever close it. The in-flight check now closes a + /// Dispatching row with NO expected-ack rows once it is past the apply deadline, so the next + /// StartDeployment is Accepted (and seals — positive control on the pipeline). + /// + [Fact] + public async Task Lost_dispatch_committed_after_PreStart_is_closed_by_the_next_StartDeployment() + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, "site-a-1:4053"); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart saw nothing + + var stale = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + dbFactory, coordinator, Enumerable.Empty(), applyDeadline: NoTimeout)); + + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var result = ExpectMsg(PresenceBudget); + result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, result.Message); + + using (var db = dbFactory.CreateDbContext()) + { + var row = db.Deployments.Single(d => d.DeploymentId == stale.Value); + row.Status.ShouldBe(DeploymentStatus.TimedOut); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("lost"); + } + + coordinator.Tell(new ApplyAck(result.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == result.DeploymentId.Value.Value).Status + .ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// Control for the closure: the same row INSIDE the deadline is still in flight and still refuses. + [Fact] + public void Dispatching_row_inside_the_deadline_still_refuses_a_new_deployment() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe(); + var pending = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(-5)); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + dbFactory, coordinator.Ref, Enumerable.Empty(), applyDeadline: NoTimeout)); + + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var result = ExpectMsg(PresenceBudget); + result.Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight); + result.DeploymentId.ShouldBe(pending); + + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching); + } + + /// + /// Fix round 1 minor — a late or duplicated DispatchDeployment for a row that is already + /// terminal (it raced the lost-dispatch closure above, or a re-delivery after a seal) must not flip + /// it back to AwaitingApplyAcks or seed an expected set. Positive control: a live row dispatched right + /// after it IS seeded (so the first message was processed, not merely not-yet-processed). + /// + [Theory] + [InlineData(DeploymentStatus.Sealed)] + [InlineData(DeploymentStatus.TimedOut)] + [InlineData(DeploymentStatus.PartiallyFailed)] + public void Dispatch_of_a_terminal_row_is_ignored(DeploymentStatus terminal) + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, "site-a-1:4053"); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + var done = SeedDeployment(dbFactory, terminal, DateTime.UtcNow.AddMinutes(-1)); + coordinator.Tell(new DispatchDeployment(done, TestRevision, CorrelationId.NewId())); + var live = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == live.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + using var verify = dbFactory.CreateDbContext(); + verify.Deployments.Single(d => d.DeploymentId == done.Value).Status.ShouldBe(terminal); + verify.NodeDeploymentStates.Count(s => s.DeploymentId == done.Value).ShouldBe(0); + } + + /// + /// Fix round 1 minor — status transitions update a blob-free stub (key + RowVersion) instead of + /// materialising the multi-MB row. Proven on the REAL provider, where the stub's RowVersion is the + /// optimistic-concurrency predicate: dispatch → ack → Sealed lands, and the artifact bytes are + /// untouched by the partial UPDATEs. + /// + [Fact] + [Trait("Category", "SchemaCompliance")] + public void On_sql_server_status_transitions_do_not_touch_the_artifact_blob() + { + using var sql = new SqlServerTestDatabase(); + var factory = sql.Factory(); + SeedNodes(factory, "site-a-1:4053"); + var blob = Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray(); + var id = DeploymentId.NewId(); + using (var db = sql.NewContext()) + { + db.Deployments.Add(new Deployment + { + DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = DeploymentStatus.Dispatching, + CreatedBy = "test", ArtifactBlob = blob, + }); + db.SaveChanges(); + } + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(factory, NoTimeout)); + AwaitAssert(() => + { + using var db = sql.NewContext(); + db.Deployments.AsNoTracking().Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); // PreStart re-drives the un-dispatched row + coordinator.Tell(new ApplyAck(id, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = sql.NewContext(); + var row = db.Deployments.AsNoTracking().Single(); + row.Status.ShouldBe(DeploymentStatus.Sealed); + row.SealedAtUtc.ShouldNotBeNull(); + row.ArtifactBlob.ShouldBe(blob); + }, duration: PresenceBudget); + } + + /// + /// Re-review 1 — the reopen race. AdminOps' lost-dispatch closure commits TimedOut AFTER the coordinator's + /// terminal guard read the row as Dispatching but BEFORE the AwaitingApplyAcks flip. The flip is + /// conditioned on the guard's RowVersion, so on SQL Server it becomes a concurrency conflict and the row + /// stays TimedOut with no expected set. Positive control: the coordinator is not wedged — the next live + /// dispatch is seeded and flipped. + /// + [Fact] + [Trait("Category", "SchemaCompliance")] + public async Task On_sql_server_a_closure_between_guard_and_flip_is_not_reopened() + { + using var sql = new SqlServerTestDatabase(); + var inner = sql.Factory(); + SeedNodes(inner, "site-a-1:4053"); + var interposer = new InterposingDbFactory(inner); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(interposer, NoTimeout)); + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart ran over an empty table + + // The race proper: a new Dispatching row whose dispatch is interleaved with an AdminOps-style closure. + var victim = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow); + interposer.OnSecondCreate(() => + { + using var db = sql.NewContext(); + db.Database.ExecuteSqlRaw( + "UPDATE Deployment SET Status = {0}, FailureReason = N'closed as lost' WHERE DeploymentId = {1}", + (int)DeploymentStatus.TimedOut, victim.Value); + }); + coordinator.Tell(new DispatchDeployment(victim, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => interposer.Fired.ShouldBeTrue(), duration: PresenceBudget); + + // Positive control + ordering: a live row dispatched after it IS flipped, so the victim's dispatch was processed. + var live = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow); + coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = sql.NewContext(); + db.Deployments.AsNoTracking().Single(d => d.DeploymentId == live.Value).Status + .ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + using var verify = sql.NewContext(); + verify.Deployments.AsNoTracking().Single(d => d.DeploymentId == victim.Value).Status + .ShouldBe(DeploymentStatus.TimedOut, "the closure must not be reopened"); + verify.NodeDeploymentStates.Count(s => s.DeploymentId == victim.Value).ShouldBe(0); + } + + /// Runs an action just before handing out the SECOND context after arming (HandleDispatch: guard, then flip). + private sealed class InterposingDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private Action? _action; + private int _countdown; + public volatile bool Fired; + + public void OnSecondCreate(Action action) + { + _action = action; + Volatile.Write(ref _countdown, 2); + } + + public OtOpcUaConfigDbContext CreateDbContext() + { + if (Volatile.Read(ref _countdown) > 0 && Interlocked.Decrement(ref _countdown) == 0) + { + _action?.Invoke(); + Fired = true; + } + return inner.CreateDbContext(); + } + } + + private static DeploymentId SeedDeployment( + IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) + { + var id = DeploymentId.NewId(); + using var db = dbFactory.CreateDbContext(); + db.Deployments.Add(new Deployment + { + DeploymentId = id.Value, + RevisionHash = TestRevision.Value, + Status = status, + CreatedBy = "test", + CreatedAtUtc = createdAtUtc, + }); + db.SaveChanges(); + return id; + } + + private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodeIds) + { + using var db = dbFactory.CreateDbContext(); + db.ServerClusters.Add(new ServerCluster + { + ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", + NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + foreach (var nodeId in nodeIds) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0], + ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test", + }); + } + db.SaveChanges(); + } + + /// Throws from exactly one CreateDbContext call after . + private sealed class OneShotFaultingDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private int _armed; + public bool Faulted { get; private set; } + + public void FailNext() => Interlocked.Exchange(ref _armed, 1); + + public OtOpcUaConfigDbContext CreateDbContext() + { + if (Interlocked.Exchange(ref _armed, 0) == 1) + { + Faulted = true; + throw new InvalidOperationException("injected transient fault seeding the expected set"); + } + return inner.CreateDbContext(); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs new file mode 100644 index 00000000..bb607aaf --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs @@ -0,0 +1,233 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:1.2 — every deploy stored the whole fleet snapshot forever. The retention sweep empties the +/// blob on old terminal rows (never deletes a row, never touches an in-flight one, never the newest N, +/// never one a node applied recently). +/// +public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase +{ + private static readonly DateTime SweepNow = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc); + + [Fact] + public async Task Sweep_empties_blobs_beyond_the_keep_window_and_leaves_the_newest_intact() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 12, mixInTimedOut: false); // ids[0] newest … ids[11] oldest + var options = new DeploymentRetentionOptions { KeepSealed = 5, ReferencedDays = 30, MaxRowsPerSweep = 200 }; + + int pruned; + await using (var db = f.CreateDbContext()) + pruned = await DeploymentRetentionActor.SweepAsync(db, options, SweepNow); + + pruned.ShouldBe(7); + await using var verify = f.CreateDbContext(); + foreach (var id in ids.Take(5)) + (await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.Length.ShouldBeGreaterThan(0, "newest 5 keep their blob"); + foreach (var id in ids.Skip(5)) + (await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.ShouldBeEmpty("older rows are emptied"); + // Rows survive: the audit record is intact. + (await verify.Deployments.CountAsync()).ShouldBe(12); + + // Idempotent: a second sweep finds nothing. + await using var again = f.CreateDbContext(); + (await DeploymentRetentionActor.SweepAsync(again, options, SweepNow)).ShouldBe(0); + } + + [Fact] + public async Task Sweep_keeps_a_row_a_node_applied_recently_and_never_touches_an_in_flight_row() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 4); + var inFlight = Guid.NewGuid(); + await using (var db = f.CreateDbContext()) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1", + ApplicationUri = "urn:x", Enabled = true, CreatedBy = "test", + }); + // ids[3] is the oldest; a node applied it yesterday, so it stays. + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = "site-a-1:4053", DeploymentId = ids[3], Status = NodeDeploymentStatus.Applied, + StartedAtUtc = SweepNow.AddDays(-1), AppliedAtUtc = SweepNow.AddDays(-1), + }); + // ids[2]: applied long ago — not protected. + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = "site-a-1:4053", DeploymentId = ids[2], Status = NodeDeploymentStatus.Applied, + StartedAtUtc = SweepNow.AddDays(-90), AppliedAtUtc = SweepNow.AddDays(-90), + }); + // An ancient in-flight row with a blob — never touched. + db.Deployments.Add(new Deployment + { + DeploymentId = inFlight, RevisionHash = new string('f', 64), Status = DeploymentStatus.AwaitingApplyAcks, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-400), ArtifactBlob = new byte[] { 1, 2, 3 }, + }); + await db.SaveChangesAsync(); + } + var options = new DeploymentRetentionOptions { KeepSealed = 1, ReferencedDays = 30 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2); // ids[1], ids[2] + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty("newest"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[1])).ArtifactBlob.ShouldBeEmpty(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[2])).ArtifactBlob.ShouldBeEmpty("applied 90 days ago"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[3])).ArtifactBlob.ShouldNotBeEmpty("applied yesterday"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == inFlight)).ArtifactBlob.ShouldNotBeEmpty("in flight"); + } + + /// + /// Fix round 1 / I3 — the keep window counts SEALED rows, not "terminal" ones. A burst of 60 failed + /// deploys used to fill a 50-row terminal window on its own and push the fleet's last good (Sealed) + /// artifact out of it — the one every boot / drift / fallback path reads. The newest Sealed blob is + /// never pruned; the unreferenced TimedOut rows are (positive control: the sweep did run). + /// + [Fact] + public async Task Sixty_timed_out_deploys_never_prune_the_newest_sealed_blob() + { + var f = NewInMemoryDbFactory(); + var sealedId = Guid.NewGuid(); + var timedOut = new List(); + await using (var db = f.CreateDbContext()) + { + db.Deployments.Add(new Deployment + { + DeploymentId = sealedId, RevisionHash = new string('s', 64), Status = DeploymentStatus.Sealed, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-200), SealedAtUtc = SweepNow.AddDays(-200), + ArtifactBlob = new byte[] { 1, 2, 3 }, + }); + for (var i = 0; i < 60; i++) + { + var id = Guid.NewGuid(); + timedOut.Add(id); + db.Deployments.Add(new Deployment + { + DeploymentId = id, RevisionHash = new string('t', 64), Status = DeploymentStatus.TimedOut, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddHours(-i), ArtifactBlob = new byte[] { 9 }, + }); + } + await db.SaveChangesAsync(); + } + var options = new DeploymentRetentionOptions { KeepSealed = 50, ReferencedDays = 30, MaxRowsPerSweep = 200 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(60); + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == sealedId)).ArtifactBlob + .ShouldNotBeEmpty("the newest Sealed artifact is never pruned"); + (await verify.Deployments.CountAsync(d => timedOut.Contains(d.DeploymentId) && d.ArtifactBlob.Length == 0)) + .ShouldBe(60); + } + + /// Even with KeepSealed misconfigured to 0, the newest Sealed row keeps its blob. + [Fact] + public async Task The_newest_sealed_blob_survives_a_zero_keep_window() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 3, mixInTimedOut: false); + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, new DeploymentRetentionOptions { KeepSealed = 0 }, SweepNow)) + .ShouldBe(2); + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty(); + } + + [Fact] + public async Task Sweep_is_bounded_per_call_and_drains_a_backlog_over_calls() + { + var f = NewInMemoryDbFactory(); + Seed(f, sealedCount: 10); + var options = new DeploymentRetentionOptions { KeepSealed = 2, MaxRowsPerSweep = 3 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(0); + } + + /// The actor runs the same rule off its thread and reports back. + [Fact] + public void Actor_sweeps_on_request_and_reports_the_count() + { + var f = NewInMemoryDbFactory(); + Seed(f, sealedCount: 6); + var actor = Sys.ActorOf(DeploymentRetentionActor.Props(f, new DeploymentRetentionOptions + { + KeepSealed = 4, InitialDelaySeconds = 3600, IntervalMinutes = 60, + })); + + actor.Tell(DeploymentRetentionActor.Sweep.Instance); + + ExpectMsg(PresenceBudget).Pruned.ShouldBe(2); + using var verify = f.CreateDbContext(); + verify.Deployments.Count(d => d.ArtifactBlob.Length == 0).ShouldBe(2); + } + + [Fact] + public void Options_bind_from_configuration_with_defaults() + { + var defaults = DeploymentRetentionOptions.Bind(null); + defaults.Enabled.ShouldBeTrue(); + defaults.KeepSealed.ShouldBe(50); + defaults.ReferencedDays.ShouldBe(30); + defaults.IntervalMinutes.ShouldBe(60); + + var config = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Deployments:Retention:KeepSealed"] = "7", + ["Deployments:Retention:Enabled"] = "false", + }) + .Build(); + var bound = DeploymentRetentionOptions.Bind(config); + bound.KeepSealed.ShouldBe(7); + bound.Enabled.ShouldBeFalse(); + bound.ReferencedDays.ShouldBe(30); + } + + /// Seeds sealed deployments, newest first; each one hour older than the previous. + private static List Seed(IDbContextFactory f, int sealedCount, bool mixInTimedOut = true) + { + var ids = new List(); + using var db = f.CreateDbContext(); + for (var i = 0; i < sealedCount; i++) + { + var id = Guid.NewGuid(); + ids.Add(id); + db.Deployments.Add(new Deployment + { + DeploymentId = id, + RevisionHash = new string('a', 64), + Status = mixInTimedOut && i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed, + CreatedBy = "test", + CreatedAtUtc = SweepNow.AddHours(-i), + SealedAtUtc = SweepNow.AddHours(-i), + ArtifactBlob = new byte[] { 0x1f, 0x8b, (byte)i }, + }); + } + db.SaveChanges(); + return ids; + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs new file mode 100644 index 00000000..3415bb99 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs @@ -0,0 +1,77 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +/// +/// A migrated, throw-away config database on the central SQL Server, for the few ControlPlane behaviours +/// the InMemory provider cannot model (a SQL Server DbUpdateException names the WHOLE statement +/// batch; collation-ordered reads). Same server / credentials / env overrides as +/// Configuration.Tests.SchemaComplianceFixture (OTOPCUA_CONFIG_TEST_SERVER, +/// OTOPCUA_CONFIG_TEST_SA_PASSWORD; default the shared docker host's dev SA), and the same database +/// name shape, so that suite's 24 h orphan reaper also reclaims a database a killed run leaks here. +/// +public sealed class SqlServerTestDatabase : IDisposable +{ + private const string DefaultServer = "10.100.0.35,14330"; + private const string DefaultSaPassword = "OtOpcUaDev_2026!"; + + /// Gets the connection string of the migrated database. + public string ConnectionString { get; } + + private readonly string _databaseName; + + /// Creates and migrates the database. + public SqlServerTestDatabase() + { + var server = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SERVER") ?? DefaultServer; + var saPassword = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SA_PASSWORD") ?? DefaultSaPassword; + _databaseName = $"OtOpcUaConfig_Test_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}"; + ConnectionString = + $"Server={server};Database={_databaseName};User Id=sa;Password={saPassword};TrustServerCertificate=True;Encrypt=False;"; + using var ctx = NewContext(); + ctx.Database.Migrate(); + } + + /// A new context on the database. + /// The context. + public OtOpcUaConfigDbContext NewContext() => + new(new DbContextOptionsBuilder().UseSqlServer(ConnectionString).Options); + + /// A factory over . + /// The factory. + public IDbContextFactory Factory() => new ContextFactory(this); + + /// Drops the database (best effort; a leak is reaped by the Configuration suite). + public void Dispose() + { + try + { + SqlConnection.ClearAllPools(); + var master = new SqlConnectionStringBuilder(ConnectionString) { InitialCatalog = "master" }.ConnectionString; + using var conn = new SqlConnection(master); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = @" +DECLARE @sql nvarchar(max); +IF DB_ID(@name) IS NOT NULL +BEGIN + SET @sql = N'ALTER DATABASE ' + QUOTENAME(@name) + N' SET SINGLE_USER WITH ROLLBACK IMMEDIATE; ' + + N'DROP DATABASE ' + QUOTENAME(@name) + N';'; + EXEC sp_executesql @sql; +END"; + cmd.Parameters.Add(new SqlParameter("@name", _databaseName)); + cmd.ExecuteNonQuery(); + } + catch (Exception) + { + // Best effort — see the class remarks. + } + } + + private sealed class ContextFactory(SqlServerTestDatabase owner) : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => owner.NewContext(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs new file mode 100644 index 00000000..cd18ae5e --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs @@ -0,0 +1,64 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Host.Engines; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// runs every compiled expression through a +/// keyed by virtual-tag id: a runaway expression leaks one orphaned pool +/// thread, not one per trigger, later triggers are refused, the tag is quarantined after three consecutive timeouts (surfacing +/// as a Failure the actor publishes Bad for), and the quarantine lifts at the deploy boundary. +/// +public sealed class RoslynVirtualTagEvaluatorRunGuardTests +{ + private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger()); + + private const string SpinScript = + "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;"; + + private static readonly Dictionary NoDeps = new(); + + [Fact] + public void Runaway_expression_is_refused_while_its_orphan_runs_and_a_sibling_keeps_evaluating() + { + using var sut = new RoslynVirtualTagEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60)); + + var first = sut.Evaluate("vt-loop", SpinScript, NoDeps); + first.Success.ShouldBeFalse(); + first.Reason.ShouldNotBeNull(); + first.Reason.ShouldContain("timed out"); + + // Refusals, not strikes (fix round 1, I1): the tag stays Bad but is not quarantined by them. + sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + sut.QuarantinedScripts.ShouldBeEmpty(); + sut.RunGuard.StartedRunCount.ShouldBe(1, "one orphan, not one per trigger"); + + var sibling = sut.Evaluate("vt-ok", "return 42;", NoDeps); + sibling.Success.ShouldBeTrue(sibling.Reason); + sibling.Value.ShouldBe(42); + } + + [Fact] + public void ResetQuarantines_lifts_the_quarantine_as_a_redeploy_does() + { + using var sut = new RoslynVirtualTagEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60), + new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + + sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + sut.QuarantinedScripts.ShouldNotBeEmpty(); + + sut.ResetQuarantines(); + + sut.QuarantinedScripts.ShouldBeEmpty(); + var fixedRun = sut.Evaluate("vt-loop", "return 7;", NoDeps); + fixedRun.Success.ShouldBeTrue(fixedRun.Reason); + fixedRun.Value.ShouldBe(7); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs index 004b7165..47ac617a 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs @@ -489,12 +489,18 @@ public sealed class RoslynVirtualTagEvaluatorTests // most evaluations then hit the warm cache instead of recompiling. Neither the cadence nor the // iteration count is a correctness knob any more — the assertions below hold for any number of // clears landing inside a single evaluation, and for any number of evaluations completed. - var evalLoops = Enumerable.Range(0, loopCount).Select(_ => Task.Run(() => + // One virtual-tag id PER LOOP, sharing the expression. The race under test lives in the compile + // cache, which is keyed by source, so twelve ids on one expression contend for exactly the entry + // twelve copies of one id did — while twelve concurrent evaluations of ONE id is a shape production + // never produces (one VirtualTagActor per id serialises them) and exactly the shape the per-tag + // ScriptRunGuard refuses ("previous evaluation still running", then quarantine). + var evalLoops = Enumerable.Range(0, loopCount).Select(loop => Task.Run(() => { + var vtagId = $"vt-race-{loop}"; var done = 0; for (var i = 0; i < MaxIterationsPerLoop && Environment.TickCount64 < stopAtMs; i++) { - var r = sut.Evaluate("vt-race", expr, new Dictionary { ["a"] = i }); + var r = sut.Evaluate(vtagId, expr, new Dictionary { ["a"] = i }); done++; if (!r.Success) { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs new file mode 100644 index 00000000..e91fbda0 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration; + +/// +/// R03:2.3 — an unset or placeholder pepper must fail host start, not surface as "Invalid API key" +/// on the first activation. +/// +public sealed class ApiKeyOptionsValidatorTests +{ + private static ValidateOptionsResult Validate(ApiKeyOptions options) => + new ApiKeyOptionsValidator().Validate(Options.DefaultName, options); + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Blank_pepper_fails(string pepper) + { + var result = Validate(new ApiKeyOptions { Pepper = pepper }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain("Pepper is required"); + } + + [Fact] + public void Short_pepper_fails() + { + var result = Validate(new ApiKeyOptions { Pepper = "x" }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain($"at least {ApiKeyOptionsValidator.MinPepperLength}"); + } + + [Fact] + public void Blank_username_fails() + { + var result = Validate(new ApiKeyOptions { Pepper = new string('p', 32), Username = " " }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain("Username"); + } + + /// Positive control: the docker-dev rig's pepper and the defaults pass unchanged. + [Fact] + public void Configured_pepper_passes() + { + Validate(new ApiKeyOptions { Pepper = "docker-dev-apikey-pepper-not-for-production" }) + .Succeeded.ShouldBeTrue(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs index 1a9730ce..45ff262b 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs @@ -49,6 +49,41 @@ public sealed class ActiveTierRegistrationTests ClusterPrimaryRegistration(hasAdmin).Tags.ShouldContain(ZbHealthTags.Active); } + /// + /// R03:2.1 + fix round 1 / I6 (coordinator ruling) — the OPC UA endpoint probe's tiers by role: + /// + /// driver-only: Ready + Active (it cannot serve and must not be routed to); + /// fused admin+driver: Ready ONLY, until the ServiceLevel calculator consumes the fact + /// (F-C1) — Traefik routes the AdminUI on /health/active, so a failed SDK on the active + /// central node must not take the AdminUI out of rotation; + /// admin-only: absent (it starts no endpoint to be degraded by). + /// + /// + [Theory] + [InlineData(false, true, true, true)] + [InlineData(true, true, true, false)] + [InlineData(true, false, false, false)] + public void Opcua_endpoint_probe_tiers_follow_the_node_roles(bool hasAdmin, bool hasDriver, bool ready, bool active) + { + var registrations = new ServiceCollection() + .AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: hasDriver) + .BuildServiceProvider() + .GetRequiredService>() + .Value.Registrations + .Where(r => r.Name == OpcUaEndpointHealthCheck.Name) + .ToList(); + + if (!ready && !active) + { + registrations.ShouldBeEmpty(); + return; + } + + var registration = registrations.ShouldHaveSingleItem(); + registration.Tags.Contains(ZbHealthTags.Ready).ShouldBe(ready); + registration.Tags.Contains(ZbHealthTags.Active).ShouldBe(active); + } + [Fact] public void Active_tier_check_is_not_on_the_ready_tier() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs new file mode 100644 index 00000000..8d08df5e --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs @@ -0,0 +1,203 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// R03:3.3 — the negative cache was keyed by key id, so a flood of DISTINCT well-formed ids cost one +/// SQL query per attempt under the SessionManager lock, no credential needed. Two guards: +/// the known-id set (unknown ids refused in memory) and the miss budget (a token bucket on lookups +/// that reach the store and miss, for when the set is unavailable). +/// +public sealed class BoundedApiKeyStoreKnownIdTests +{ + private static readonly DateTimeOffset T0 = new(2026, 10, 22, 12, 0, 0, TimeSpan.Zero); + + [Fact] + public async Task An_unknown_id_is_refused_without_reaching_the_store_and_a_known_one_still_does() + { + var inner = new ListingStore("k-real"); + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + (await sut.FindAsync("k-guess-1", T0.UtcDateTime, default)).ShouldBeNull(); + (await sut.FindAsync("k-guess-2", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(0, "unknown ids never reach the store"); + inner.ListCalls.ShouldBe(1, "one set load serves every unknown id inside the TTL"); + + // Positive control: the real id reaches the store and is found. + (await sut.FindAsync("k-real", T0.UtcDateTime, default)).ShouldNotBeNull(); + inner.FindCalls.ShouldBe(1); + } + + [Fact] + public async Task The_set_is_refreshed_after_the_TTL_so_a_newly_minted_key_becomes_visible() + { + var inner = new ListingStore("k-old"); + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + (await sut.FindAsync("k-new", T0.UtcDateTime, default)).ShouldBeNull(); + inner.Mint("k-new"); + // Still inside the TTL: the stale set refuses it (the documented window). + clock.Advance(TimeSpan.FromSeconds(3)); + (await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(0); + + clock.Advance(TimeSpan.FromSeconds(3)); + (await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + inner.ListCalls.ShouldBe(2); + } + + [Fact] + public async Task A_failed_set_load_falls_back_to_the_per_id_lookup_and_keeps_an_older_set() + { + var inner = new ListingStore("k-real") { ListThrows = true }; + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + // No set yet and the load fails: the per-id path runs (old behaviour), nothing is wedged. + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(1); + sut.KnownKeyIds.ShouldBeNull(); + + inner.ListThrows = false; + clock.Advance(TimeSpan.FromSeconds(6)); + (await sut.FindAsync("k-guess-2", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + sut.KnownKeyIds.ShouldNotBeNull(); + inner.FindCalls.ShouldBe(1, "once the set loaded, the guess was refused in memory"); + + // A later failed refresh keeps the previous set in force. + inner.ListThrows = true; + clock.Advance(TimeSpan.FromSeconds(6)); + (await sut.FindAsync("k-guess-3", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(1); + } + + /// With no id set available, misses are rate-limited; hits are not charged; time refills. + [Fact] + public async Task The_miss_budget_bounds_store_misses_when_no_set_is_available() + { + var inner = new ListingStore("k-real") { SupportsListing = false }; + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 0, missBurst: 3, refillPerSecond: 1); + + for (var i = 0; i < 3; i++) + (await sut.FindAsync($"k-guess-{i}", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(3); + + // Budget exhausted: refused without a store read, as a throw the authenticator denies on. + await Should.ThrowAsync(() => sut.FindAsync("k-guess-9", T0.UtcDateTime, default)); + inner.FindCalls.ShouldBe(3); + + // Refill: one second restores one token. + clock.Advance(TimeSpan.FromSeconds(1)); + (await sut.FindAsync("k-guess-10", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(4); + + // A hit is not a miss: it reaches the store even with the bucket empty? No — it needs a token to + // be attempted, but the token is refunded, so the next attempt still has it. Prove the refund. + clock.Advance(TimeSpan.FromSeconds(1)); + (await sut.FindAsync("k-real", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + (await sut.FindAsync("k-guess-11", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(6); + } + + [Fact] + public async Task The_guards_are_off_when_the_cache_is_off_restoring_a_read_per_lookup() + { + var inner = new ListingStore("k-real"); + var sut = Bounded(inner, new FakeClock(T0), cacheSeconds: 0, missBurst: 0); + + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(2); + inner.ListCalls.ShouldBe(0); + } + + /// + /// Fix round 1 minor — an inner store that cannot list ids switches the known-id set off for THIS + /// decorator only. It used to do so by writing KnownIdSet = false into the guard options it was + /// handed, a shared options object, so one store's capability leaked into every other consumer. + /// Positive control: a second decorator over a listing store on the same options instance still uses + /// the set. + /// + [Fact] + public async Task An_unlisting_store_does_not_mutate_the_shared_guard_options() + { + var guard = new ApiKeyStoreGuardOptions { MissBurst = 100, MissRefillPerSecond = 20 }; + var options = Options.Create(new ApiKeyOptions + { + Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = 5, + }); + var clock = new FakeClock(T0); + var unlisting = new ListingStore("k-real") { SupportsListing = false }; + var first = new BoundedApiKeyStore(unlisting, options, NullLogger.Instance, clock, guard); + + (await first.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + unlisting.FindCalls.ShouldBe(1, "no set: the per-id path ran"); + guard.KnownIdSet.ShouldBeTrue("the shared options object is never written"); + + var listing = new ListingStore("k-real"); + var second = new BoundedApiKeyStore(listing, options, NullLogger.Instance, clock, guard); + (await second.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + listing.FindCalls.ShouldBe(0, "the second decorator still refuses unknown ids in memory"); + listing.ListCalls.ShouldBe(1); + } + + private static BoundedApiKeyStore Bounded( + IApiKeyStore inner, TimeProvider clock, int cacheSeconds, int missBurst = 100, double refillPerSecond = 20) => new( + inner, + Options.Create(new ApiKeyOptions + { + Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = cacheSeconds, + }), + NullLogger.Instance, + clock, + new ApiKeyStoreGuardOptions { MissBurst = missBurst, MissRefillPerSecond = refillPerSecond }); + + private sealed class ListingStore(params string[] ids) : IApiKeyStore + { + private readonly HashSet _ids = ids.ToHashSet(StringComparer.Ordinal); + private int _findCalls; + private int _listCalls; + + public bool SupportsListing { get; set; } = true; + public bool ListThrows { get; set; } + public int FindCalls => _findCalls; + public int ListCalls => _listCalls; + + public void Mint(string id) => _ids.Add(id); + + public Task FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct) + { + Interlocked.Increment(ref _findCalls); + return Task.FromResult(_ids.Contains(apiKeyId) + ? new ApiKeyLookup(new ApiKey { ApiKeyId = apiKeyId, Name = "k", SecretHash = "h", Principal = "p" }, []) + : null); + } + + public Task> FindGrantsAsync(string principal, CancellationToken ct) => + Task.FromResult>([]); + + public Task?> ListKeyIdsAsync(CancellationToken ct) + { + if (!SupportsListing) return Task.FromResult?>(null); + Interlocked.Increment(ref _listCalls); + if (ListThrows) throw new InvalidOperationException("ConfigDb unreachable (simulated)"); + return Task.FromResult?>(_ids.ToHashSet(StringComparer.Ordinal)); + } + } + + private sealed class FakeClock(DateTimeOffset utcNow) : TimeProvider + { + private DateTimeOffset _now = utcNow; + public override DateTimeOffset GetUtcNow() => _now; + public void Advance(TimeSpan by) => _now = _now.Add(by); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs index e8081db1..65201905 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs @@ -363,7 +363,10 @@ public sealed class BoundedApiKeyStoreTests { var (_, inner) = SetUp(); var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); - var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock); + // The miss budget (R03:3.3) is OFF here on purpose: this fact floods 1025 distinct unknown ids — the + // exact shape the budget refuses — to pin the cache CEILING, which is a separate rule. + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock, + guard: new ApiKeyStoreGuardOptions { MissBurst = 0 }); // Seed the real key, then flood past the ceiling with ids that do not exist. (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); @@ -477,7 +480,8 @@ public sealed class BoundedApiKeyStoreTests int timeoutMs, int cacheSeconds, TimeProvider? clock = null, - ILogger? log = null) => new( + ILogger? log = null, + ApiKeyStoreGuardOptions? guard = null) => new( inner, Options.Create(new ApiKeyOptions { @@ -487,7 +491,8 @@ public sealed class BoundedApiKeyStoreTests StoreCacheSeconds = cacheSeconds, }), log ?? NullLogger.Instance, - clock ?? TimeProvider.System); + clock ?? TimeProvider.System, + guard); private static ApiKeyOpcUaUserAuthenticator Auth(IApiKeyStore store) => new( store, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs new file mode 100644 index 00000000..341595d7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs @@ -0,0 +1,213 @@ +using System.Net; +using System.Net.Sockets; +using Akka.Hosting; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.Health; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// R03:2.1 / 1.4 — a failed OPC UA SDK start used to be one Error line: the node stayed Ready, joined +/// its mesh, and acked deploys through the Null sink while clients got connection-refused. The start +/// outcome is now a fact () the ready/active tiers read, a driver-only +/// node rethrows so the process exits, and StopAsync actually stops the SDK. +/// +public sealed class OtOpcUaServerHostedServiceStartFailureTests +{ + private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30); + + /// + /// Port already bound ⇒ the SDK cannot listen ⇒ the state is Failed, the probe is Unhealthy and the + /// real /health/ready endpoint answers 503 — while StartAsync itself returns (fused node). + /// Positive control: the same endpoint answers 200 once the state is Started. + /// + [Fact] + public async Task Port_in_use_marks_the_endpoint_failed_and_health_ready_answers_503() + { + using var deadline = new CancellationTokenSource(Deadline); + using var squatter = SquatAllInterfaces(out var port); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var fused = EffectiveRoles.Resolve(new[] { "admin", "driver" }, envRoles: null); + + var service = NewService(port, pki, state, fused); + try + { + await service.StartAsync(deadline.Token); // must NOT throw on a fused node + + state.Status.ShouldBe(OpcUaEndpointStatus.Failed); + state.FailureReason.ShouldNotBeNullOrWhiteSpace(); + + var probe = await new OpcUaEndpointHealthCheck(state).CheckHealthAsync(new HealthCheckContext()); + probe.Status.ShouldBe(HealthStatus.Unhealthy); + probe.Description.ShouldNotBeNull(); + probe.Description.ShouldContain("failed to start"); + + await using var web = BuildHealthHost(state); + await web.StartAsync(deadline.Token); + using var http = new HttpClient { BaseAddress = new Uri(web.Urls.First()) }; + + (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + (await http.GetAsync("/health/active", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + + // Positive control on the endpoint: the same host answers 200 for a started endpoint, so the + // 503 above measured the probe and not a broken health pipeline. + state.MarkStarted(); + (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.OK); + await web.StopAsync(deadline.Token); + } + finally + { + await service.DisposeAsync(); + squatter.Stop(); + Cleanup(pki); + } + } + + /// A driver-only node has nothing else to stay up for: the failed start propagates. + [Fact] + public async Task Driver_only_node_rethrows_when_the_sdk_fails_to_start() + { + using var deadline = new CancellationTokenSource(Deadline); + using var squatter = SquatAllInterfaces(out var port); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var driverOnly = EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null); + + var service = NewService(port, pki, state, driverOnly); + try + { + await Should.ThrowAsync(() => service.StartAsync(deadline.Token)); + state.Status.ShouldBe(OpcUaEndpointStatus.Failed); + } + finally + { + await service.DisposeAsync(); + squatter.Stop(); + Cleanup(pki); + } + } + + /// + /// A clean start is Started and listening; StopAsync closes the port (connection refused, not + /// a stale-but-listening endpoint) and marks Stopped; DisposeAsync twice afterwards is a no-op. + /// + [Fact] + public async Task StopAsync_closes_the_port_and_DisposeAsync_is_idempotent() + { + using var deadline = new CancellationTokenSource(Deadline); + var port = FreePort(); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var service = NewService(port, pki, state, EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null)); + try + { + await service.StartAsync(deadline.Token); + state.Status.ShouldBe(OpcUaEndpointStatus.Started); + using (var client = new TcpClient()) + { + await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token); + client.Connected.ShouldBeTrue("positive control: the endpoint is listening after StartAsync"); + } + + await service.StopAsync(deadline.Token); + + state.Status.ShouldBe(OpcUaEndpointStatus.Stopped); + using (var client = new TcpClient()) + { + await Should.ThrowAsync(async () => await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token)); + } + + await service.DisposeAsync(); + await service.DisposeAsync(); // idempotent + } + finally + { + await service.DisposeAsync(); + Cleanup(pki); + } + } + + private static OtOpcUaServerHostedService NewService(int port, string pki, OpcUaEndpointState state, EffectiveRoles roles) => + new( + Options.Create(new OpcUaApplicationHostOptions + { + ApplicationName = "OtOpcUa.StartFailure", + ApplicationUri = $"urn:OtOpcUa.StartFailure:{Guid.NewGuid():N}", + OpcUaPort = port, PublicHostname = "localhost", PkiStoreRoot = pki, + }), + new DeferredAddressSpaceSink(), new DeferredServiceLevelPublisher(), + NullOpcUaUserAuthenticator.Instance, NullHistorianDataSource.Instance, + EmptyDataPlaneAuthoritySource.Instance, new ConfigurationBuilder().Build(), + () => throw new NotSupportedException("no actor system required"), new ActorRegistry(), + null, Options.Create(new AclOptions()), TimeProvider.System, NullLoggerFactory.Instance, + endpointState: state, roles: roles); + + /// The production health registration (driver role) over the supplied state, on a loopback Kestrel. + private static WebApplication BuildHealthHost(OpcUaEndpointState state) + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + builder.Logging.ClearProviders(); + builder.Services.AddSingleton(state); + builder.Services.AddOtOpcUaHealth(hasAdmin: false, hasDriver: true); + // The akka + cluster-primary probes need a running ActorSystem; keep only the probe under test so + // the status measured is the endpoint's. (The registration shape itself is pinned separately.) + builder.Services.Configure(o => + { + foreach (var r in o.Registrations.Where(r => r.Name != OpcUaEndpointHealthCheck.Name).ToList()) + o.Registrations.Remove(r); + }); + var app = builder.Build(); + app.MapOtOpcUaHealth(); + return app; + } + + /// + /// Holds a port on the IPv4 AND IPv6 wildcard addresses. The SDK binds its listener on the + /// wildcards, and on macOS a loopback-only squatter does not conflict with a wildcard bind (both + /// sockets carry SO_REUSEADDR), so a loopback squatter would let the SDK start — the first run of + /// this test proved exactly that. The SDK also only fails when NEITHER family binds. + /// + private static TcpListener SquatAllInterfaces(out int port) + { + var squatter = new TcpListener(IPAddress.IPv6Any, 0); + squatter.Server.DualMode = true; + squatter.ExclusiveAddressUse = true; + squatter.Start(); + port = ((IPEndPoint)squatter.LocalEndpoint).Port; + return squatter; + } + + private static int FreePort() + { + using var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + var port = ((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); + return port; + } + + private static string NewPki() => Path.Combine(Path.GetTempPath(), $"otopcua-startfail-{Guid.NewGuid():N}"); + + private static void Cleanup(string pki) + { + if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs index d020d118..a3861648 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs @@ -15,6 +15,28 @@ public sealed class DependencyMuxTagUpstreamSourceTests private static DataValueSnapshot Good(object? value, DateTime ts) => new(value, 0u, ts, ts); + /// R02:L3 — the host prunes the retained snapshots to the loaded dependency-ref union after each + /// (re)load, so a renamed/removed raw path stops being retained; observers are untouched. + [Fact] + public void RetainOnly_drops_snapshots_outside_the_set_and_keeps_observers() + { + var src = new DependencyMuxTagUpstreamSource(); + var ts = DateTime.UtcNow; + src.Push("keep", Good(1, ts)); + src.Push("old/renamed", Good(2, ts)); + var seen = new List(); + using var sub = src.SubscribeTag("old/renamed", (p, _) => seen.Add(p)); + + src.RetainOnly(new HashSet(StringComparer.Ordinal) { "keep" }).ShouldBe(1); + + src.RetainedPathCount.ShouldBe(1); + src.ReadTag("keep").Value.ShouldBe(1); + src.ReadTag("old/renamed").StatusCode.ShouldBe(OpcUaStatus.BadWaitingForInitialData); + src.Push("old/renamed", Good(3, ts)); + // Pruning retained values does not unsubscribe anyone. + seen.ShouldBe(new[] { "old/renamed" }); + } + [Fact] public void Push_then_ReadTag_returns_the_pushed_snapshot() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs new file mode 100644 index 00000000..c4063d57 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs @@ -0,0 +1,145 @@ +using Akka.Actor; +using Akka.TestKit; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; +using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.ScriptedAlarms; + +/// +/// R02:H1 — a supervision restart of used to dispose the +/// Props-captured engine and re-create the actor around the disposed instance, so every later +/// ApplyScriptedAlarms failed and the node's scripted alarms silently died. With the factory +/// Props each incarnation builds its own engine; with the legacy Props a restart only detaches. Both +/// shapes must load cleanly after a throw inside a +/// handler. Also R02:H3 — an inbound alarm command's engine op is bounded. +/// +public sealed class ScriptedAlarmHostActorRestartTests : RuntimeActorTestBase +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); + + private static EquipmentScriptedAlarmPlan Plan(string id = "alm-1", string depRef = "M.T") => + new(ScriptedAlarmId: id, EquipmentId: "Plant/Line1/M", Name: "HighTemp", AlarmType: "AlarmCondition", + Severity: 800, MessageTemplate: "condition", PredicateScriptId: $"{id}-script", + PredicateSource: $"return (int)ctx.GetTag(\"{depRef}\").Value > 90;", + DependencyRefs: new[] { depRef }, HistorizeToAveva: true, Retain: true, Enabled: true); + + private static ScriptedAlarmEngine BuildEngine(DependencyMuxTagUpstreamSource upstream, IAlarmStateStore? store = null) + { + var logger = new LoggerConfiguration().CreateLogger(); + return new ScriptedAlarmEngine(upstream, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger); + } + + private static (DependencyMuxTagUpstreamSource, ScriptedAlarmEngine) Pair() + { + var up = new DependencyMuxTagUpstreamSource(); + return (up, BuildEngine(up)); + } + + /// Apply, crash the handler (a null TagId makes Push throw → restart), apply again + /// and prove the second load registers interest and the alarm still activates. + private void RestartThenReload(IActorRef host, TestProbe publish, TestProbe mux) + { + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() })); + mux.ExpectMsg(Timeout); + + // The crash: a batch whose item has a null TagId makes the upstream Push throw out of the handler + // (the documented, deliberately unguarded path) → supervision restarts the actor. + host.Tell(new VirtualTagActor.DependencyValueBatch(new[] + { + new VirtualTagActor.DependencyValueChanged(null!, 1, DateTime.UtcNow), + })); + mux.ExpectMsg(Timeout); // PostStop of the crashed incarnation + + // A later apply must load cleanly on the restarted incarnation… + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() })); + var reg = mux.ExpectMsg(Timeout); + reg.TagRefs.ShouldContain("M.T"); + + // …and the alarm must still work end to end. + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow)); + var state = publish.ExpectMsg(Timeout); + state.AlarmNodeId.ShouldBe("alm-1"); + state.State.Active.ShouldBeTrue(); + } + + [Fact] + public void Factory_Props_survives_a_handler_throw_and_loads_cleanly_afterwards() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var built = 0; + var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, () => { built++; return Pair(); })); + + RestartThenReload(host, publish, mux); + + built.ShouldBe(2, "the restarted incarnation built its own engine"); + } + + [Fact] + public void Legacy_instance_Props_survives_a_handler_throw_because_a_restart_only_detaches() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var (upstream, engine) = Pair(); + var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, upstream, engine)); + + RestartThenReload(host, publish, mux); + } + + [Fact] + public void An_alarm_command_whose_store_hangs_is_bounded_and_the_mailbox_moves_on() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var store = new HangingSaveStore(); + var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, + () => { var up = new DependencyMuxTagUpstreamSource(); return (up, BuildEngine(up, store)); }, + commandTimeout: TimeSpan.FromMilliseconds(300))); + + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() })); + mux.ExpectMsg(Timeout); + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow)); + publish.ExpectMsg(Timeout).State.Active.ShouldBeTrue(); + + store.HangNext = true; + host.Tell(new AlarmCommand("alm-1", "Acknowledge", "alice", "ack", null)); + + // The hung save would have suspended the mailbox indefinitely; the bound releases it, so the + // next dependency value is processed and the alarm clears. + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 10, DateTime.UtcNow)); + publish.ExpectMsg(Timeout).State.Active.ShouldBeFalse(); + store.ObservedCancellation.ShouldBeTrue(); + } + + /// Store whose next SaveAsync blocks until its token is cancelled. + private sealed class HangingSaveStore : IAlarmStateStore + { + private readonly InMemoryAlarmStateStore _inner = new(); + public volatile bool HangNext; + public volatile bool ObservedCancellation; + + public Task LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct); + public Task> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct); + public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct); + + public async Task SaveAsync(AlarmConditionState state, CancellationToken ct) + { + if (HangNext) + { + HangNext = false; + try { await Task.Delay(System.Threading.Timeout.InfiniteTimeSpan, ct); } + catch (OperationCanceledException) { ObservedCancellation = true; throw; } + } + await _inner.SaveAsync(state, ct); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs new file mode 100644 index 00000000..f761edd9 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs @@ -0,0 +1,184 @@ +using System.Collections.Concurrent; +using Akka.Actor; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Calculation; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags; + +/// +/// R07:H6 — the mux retains the last value of every ref a REPLAY-OPTING subscriber +/// () is registered for, keeps it for a +/// short grace after the last such subscriber leaves, and replays it to a registration that asks. A +/// Calculation driver Restart (new driver + new adapter) therefore republishes from current inputs instead +/// of sitting BadWaitingForInitialData until every input happens to change. Fix round 1 (I2): a +/// non-opting subscriber (the historization recorder) retains nothing, retention is released after the +/// grace, and a removed source's values are evicted so its last Good is never replayed. +/// +public sealed class DependencyMuxReplayTests : RuntimeActorTestBase +{ + private static DriverInstanceActor.AttributeValueBatchPublished Burst(params (string Ref, object? Value)[] items) => + new("driver-1", items.Select(i => new DriverInstanceActor.AttributeValueItem(i.Ref, i.Value, OpcUaStatus.Good, DateTime.UtcNow)).ToArray()); + + [Fact] + public void RegisterInterest_with_replay_receives_the_retained_values_for_its_refs_only() + { + var mux = Sys.ActorOf(DependencyMuxActor.Props()); + var first = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, first.Ref, ReplayRetained: true)); + mux.Tell(Burst(("a", 1), ("b", 2), ("c", 3))); + first.ExpectMsg(PresenceBudget).Items.Count.ShouldBe(2); + + // The first subscriber goes away; retention must survive the unregister (that is the Restart shape). + mux.Tell(new DependencyMuxActor.UnregisterInterest(first.Ref)); + + var second = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "c" }, second.Ref, ReplayRetained: true)); + + var replay = second.ExpectMsg(PresenceBudget); + replay.Items.Select(i => i.TagId).ShouldBe(new[] { "a" }, "'c' was never registered by a replay-opting subscriber, so it is not retained; 'b' is not in this subscriber's refs"); + replay.Items[0].Value.ShouldBe(1); + } + + [Fact] + public void RegisterInterest_without_replay_receives_nothing_and_a_fresh_ref_replays_nothing() + { + var mux = Sys.ActorOf(DependencyMuxActor.Props()); + var seed = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, seed.Ref)); + mux.Tell(Burst(("a", 1))); + seed.ExpectMsg(PresenceBudget); + + var opted = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, opted.Ref)); // default: no replay + opted.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + + var fresh = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "never-published" }, fresh.Ref, ReplayRetained: true)); + fresh.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + } + + /// I2 — a subscriber that did NOT opt into replay (the continuous-historization recorder registers + /// every historized ref) retains nothing; retention is bounded to the refs a replay-opting subscriber named. + [Fact] + public void Only_refs_named_by_a_replay_opting_subscriber_are_retained() + { + var mux = ActorOfAsTestActorRef(DependencyMuxActor.Props()); + var recorder = CreateTestProbe(); + var calc = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "h1", "h2", "h3", "shared" }, recorder.Ref)); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "r", "shared" }, calc.Ref, ReplayRetained: true)); + mux.Tell(Burst(("h1", 1), ("h2", 2), ("h3", 3), ("r", 4), ("shared", 5))); + recorder.ExpectMsg(PresenceBudget).Items.Count.ShouldBe(4, "the recorder is still DELIVERED to"); + calc.ExpectMsg(PresenceBudget); + + mux.UnderlyingActor.RetainedRefCount.ShouldBe(2, "only 'r' and 'shared' — the recorder-only refs retain nothing"); + + var late = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "h1", "r" }, late.Ref, ReplayRetained: true)); + late.ExpectMsg(PresenceBudget) + .Items.Select(i => i.TagId).ShouldBe(new[] { "r" }, "positive control: the opted-in ref replays; 'h1' was never retained"); + + // Unregistering everyone ends up with no entries at all once the grace is over — tested below; here + // the recorder leaving changes nothing about retention. + mux.Tell(new DependencyMuxActor.UnregisterInterest(recorder.Ref)); + mux.UnderlyingActor.RetainedRefCount.ShouldBe(2); + } + + /// I2 — when the last replay-opting subscriber of a ref leaves, its retained value survives a short + /// grace (the restart window: old adapter gone, new one not yet registered) and is then dropped. + [Fact] + public void Retention_is_released_after_the_grace_once_no_replay_subscriber_remains() + { + var mux = ActorOfAsTestActorRef(DependencyMuxActor.Props(TimeSpan.FromMilliseconds(200))); + var gen1 = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen1.Ref, ReplayRetained: true)); + mux.Tell(Burst(("a", 1))); + gen1.ExpectMsg(PresenceBudget); + mux.Tell(new DependencyMuxActor.UnregisterInterest(gen1.Ref)); + + // Positive control: inside the grace a returning subscriber still gets the value. + var gen2 = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen2.Ref, ReplayRetained: true)); + gen2.ExpectMsg(PresenceBudget).Items.Single().Value.ShouldBe(1); + mux.Tell(new DependencyMuxActor.UnregisterInterest(gen2.Ref)); + + // Past the grace: nothing retained, no entry left behind, and a returning subscriber replays nothing. + AwaitAssert(() => + { + mux.UnderlyingActor.RetainedRefCount.ShouldBe(0); + mux.UnderlyingActor.TrackedRefCount.ShouldBe(0); + }, PresenceBudget); + var gen3 = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen3.Ref, ReplayRetained: true)); + gen3.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + } + + /// I2 — a removed source's retained values are evicted, so its last Good is never replayed to a + /// subscriber that (re)registers after the removal. Values from other sources are untouched. + [Fact] + public void SourceRemoved_evicts_that_sources_retained_values_only() + { + var mux = ActorOfAsTestActorRef(DependencyMuxActor.Props()); + var sub = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, sub.Ref, ReplayRetained: true)); + mux.Tell(new DriverInstanceActor.AttributeValueBatchPublished("driver-1", + [new DriverInstanceActor.AttributeValueItem("a", 1, OpcUaStatus.Good, DateTime.UtcNow)])); + mux.Tell(new DriverInstanceActor.AttributeValueBatchPublished("driver-2", + [new DriverInstanceActor.AttributeValueItem("b", 2, OpcUaStatus.Good, DateTime.UtcNow)])); + sub.ExpectMsg(PresenceBudget); + sub.ExpectMsg(PresenceBudget); + + mux.Tell(new DependencyMuxActor.SourceRemoved("driver-1")); + mux.UnderlyingActor.RetainedRefCount.ShouldBe(1); + + var restarted = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, restarted.Ref, ReplayRetained: true)); + restarted.ExpectMsg(PresenceBudget) + .Items.Select(i => i.TagId).ShouldBe(new[] { "b" }, "driver-1 was removed: its last Good must not be replayed"); + } + + /// The user-facing scenario: a calc tag computes, the driver is restarted (new instance + new + /// adapter) with NO input changing afterwards, and the new instance republishes the current value. + [Fact] + public async Task A_Calculation_driver_restart_republishes_from_the_current_inputs() + { + var mux = Sys.ActorOf(DependencyMuxActor.Props()); + var tag = new RawTagEntry("calc/e/sum", + "{\"scriptId\":\"s\",\"changeTriggered\":true,\"scriptSource\":\"return (double)ctx.GetTag(\\\"src/a\\\").Value + (double)ctx.GetTag(\\\"src/b\\\").Value;\"}", + WriteIdempotent: false); + + // Generation 1: wire the driver through the real adapter and feed a burst. + var gen1 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv"); + var published1 = new ConcurrentQueue(); + gen1.OnDataChange += (_, e) => published1.Enqueue(e); + await gen1.InitializeAsync("{}", CancellationToken.None); + var adapter1 = Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen1, mux)); + AwaitAssert(() => + { + mux.Tell(Burst(("src/a", 10.0), ("src/b", 20.0))); + published1.Count.ShouldBeGreaterThan(0); + }, PresenceBudget); + published1.Last().Snapshot.Value.ShouldBe(30.0); + + // Restart: the old instance goes away; a NEW driver + adapter come up. Nothing changes upstream. + Sys.Stop(adapter1); + await gen1.ShutdownAsync(CancellationToken.None); + var gen2 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv"); + var published2 = new ConcurrentQueue(); + gen2.OnDataChange += (_, e) => published2.Enqueue(e); + Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen2, mux)); // registers with ReplayRetained + await gen2.InitializeAsync("{}", CancellationToken.None); + + // The replayed inputs satisfied the change-gate; the first trigger after init is the replay itself + // (if it lands post-init) or the next change (here: none) — so publish happens from the replay. + AwaitAssert(() => published2.Count.ShouldBeGreaterThan(0), PresenceBudget); + published2.Last().Snapshot.Value.ShouldBe(30.0); + published2.Last().Snapshot.StatusCode.ShouldBe(OpcUaStatus.Good); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs new file mode 100644 index 00000000..c1a6253e --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs @@ -0,0 +1,56 @@ +using System.Collections.Concurrent; +using Akka.Actor; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Engines; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags; + +/// R02:P6 — the per-failure cluster-wide is rate-limited per tag +/// (first N per window, then one summary); the Bad publish and the metric are unaffected. +public sealed class VirtualTagActorFailureLogRateLimitTests : RuntimeActorTestBase +{ + [Fact] + public void Repeated_failures_publish_at_most_the_limit_then_one_summary_when_the_window_rolls() + { + var now = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc); + var limiter = new ScriptLogRateLimiter(limitPerWindow: 3, window: TimeSpan.FromMinutes(1), clock: () => now); + var capture = new ConcurrentQueue(); + var parent = CreateTestProbe(); + var actor = parent.ChildActorOf(VirtualTagActor.Props( + "vt-chatty", "broken", + evaluator: new FailingEvaluator("boom"), + publisherFactory: () => new DPSPublisher((_, p) => capture.Enqueue((ScriptLogEntry)p)), + failureLogLimiter: limiter)); + + for (var i = 0; i < 10; i++) + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", i, DateTime.UtcNow)); + + // Positive control: the FIRST failure still degrades the node (once per transition, as before). + parent.ExpectMsg(PresenceBudget).StatusCode.ShouldBe(Commons.OpcUa.OpcUaStatus.BadInternalError); + AwaitAssert(() => capture.Count.ShouldBe(3), PresenceBudget); + parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + capture.Count.ShouldBe(3, "7 of 10 failures were suppressed from the cluster-wide log"); + + now = now.AddMinutes(1); + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 99, DateTime.UtcNow)); + + AwaitAssert(() => capture.Count.ShouldBe(5), PresenceBudget); + var entries = capture.ToArray(); + entries[3].Message.ShouldContain("7"); + entries[3].Message.ShouldContain("suppressed"); + entries[3].Message.ShouldContain("limit 3/window", Case.Sensitive, "the summary names the CONFIGURED limit, not the default"); + entries[3].VirtualTagId.ShouldBe("vt-chatty"); + entries[4].Message.ShouldBe("boom"); + } + + private sealed class FailingEvaluator(string reason) : IVirtualTagEvaluator + { + public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary deps) + => VirtualTagEvalResult.Failure(reason); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs new file mode 100644 index 00000000..6341ef6f --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs @@ -0,0 +1,116 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Components.Authorization; +using Microsoft.AspNetCore.Http; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Auth.AspNetCore; +using ZB.MOM.WW.OtOpcUa.Security.Audit; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests.Audit; + +/// +/// R03:2.5 — inside a Blazor circuit there is no HttpContext after the first render, so an +/// HTTP-only accessor stamps every audit row from an interactive page "system". The circuit's +/// is consulted first; the HTTP context is the fallback. +/// +public sealed class AuthenticationStateAuditActorAccessorTests +{ + [Fact] + public void Circuit_principal_wins_when_there_is_no_http_context() + { + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new FixedStateProvider(Authenticated(new Claim(ZbClaimTypes.Username, "alice")))); + + accessor.CurrentActor.ShouldBe("alice"); + } + + [Fact] + public void Http_principal_is_the_fallback_outside_a_circuit() + { + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "bob")) }; + var accessor = new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), authenticationState: null); + + accessor.CurrentActor.ShouldBe("bob"); + } + + [Fact] + public void Unauthenticated_circuit_falls_through_to_http_then_null() + { + var anonymousCircuit = new FixedStateProvider(new ClaimsPrincipal(new ClaimsIdentity())); + + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(null), anonymousCircuit) + .CurrentActor.ShouldBeNull(); + + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Name, "carol")) }; + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), anonymousCircuit) + .CurrentActor.ShouldBe("carol"); + } + + /// An incomplete state task is "no circuit identity", never a block. + [Fact] + public void Pending_authentication_state_is_not_awaited() + { + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new PendingStateProvider()); + + accessor.CurrentActor.ShouldBeNull(); + } + + /// + /// Fix round 1 / I5 — the REAL server-side provider, resolved in a non-circuit scope (a minimal-API + /// request, a background service), throws from + /// GetAuthenticationStateAsync because nothing called SetAuthenticationState. That used + /// to escape CurrentActor; it must fall back to the HTTP principal instead. + /// + [Fact] + public void Real_server_provider_outside_a_circuit_falls_back_to_the_http_principal() + { + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "dave")) }; + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(context), + new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider()); + + accessor.CurrentActor.ShouldBe("dave"); + + // No HTTP principal either ⇒ null, still no throw. + new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider()) + .CurrentActor.ShouldBeNull(); + } + + /// Positive control on the real provider: once a circuit has set its state, that principal wins. + [Fact] + public void Real_server_provider_inside_a_circuit_serves_the_circuit_principal() + { + var provider = new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider(); + provider.SetAuthenticationState(Task.FromResult( + new AuthenticationState(Authenticated(new Claim(ZbClaimTypes.Username, "erin"))))); + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "http-user")) }; + + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), provider) + .CurrentActor.ShouldBe("erin"); + } + + private static ClaimsPrincipal Authenticated(params Claim[] claims) => + new(new ClaimsIdentity(claims, "TestScheme", ZbClaimTypes.Name, ZbClaimTypes.Role)); + + private sealed class HttpContextAccessorStub(HttpContext? context) : IHttpContextAccessor + { + public HttpContext? HttpContext { get; set; } = context; + } + + private sealed class FixedStateProvider(ClaimsPrincipal user) : AuthenticationStateProvider + { + public override Task GetAuthenticationStateAsync() => + Task.FromResult(new AuthenticationState(user)); + } + + private sealed class PendingStateProvider : AuthenticationStateProvider + { + public override Task GetAuthenticationStateAsync() => + new TaskCompletionSource().Task; + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs index 9cf8ab11..599d0cf2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs @@ -11,6 +11,8 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; using Shouldly; using Xunit; using ZB.MOM.WW.Auth.AspNetCore; @@ -37,6 +39,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime private IHost _host = null!; private TestServer _server = null!; private readonly StubLdapGroupRoleMappingService _roleMappings = new(); + private readonly CapturingLoggerProvider _logs = new(); private static CancellationToken Ct => TestContext.Current.CancellationToken; @@ -70,6 +73,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime ["Security:Ldap:GroupToRole:ReadOnly"] = "Viewer", }).Build(); services.AddOtOpcUaAuth(configuration); + services.AddLogging(l => l.AddProvider(_logs)); services.AddSingleton(); // The login handler now resolves the DB role-map service via DI to merge // DB-backed grants on top of the appsettings baseline. Register the stub so @@ -96,6 +100,10 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime .ToArray(); return Results.Ok(claims); }).RequireAuthorization(); + // A role-gated machine endpoint: proves a bearer principal's roles resolve through + // IsInRole (the validation parameters pair RoleClaimType with the minted key). + e.MapGet("/api/viewer-only", () => Results.Ok("viewer")) + .RequireAuthorization(p => p.RequireRole("Viewer")); }); }); }) @@ -153,6 +161,158 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime new AuthEndpoints.LoginRequest("ldap-down", "anything"), Ct); response.StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + // R03:2.7 — the outage is named in the server log, not inferred from the directory side. + _logs.Entries.Any(e => + e.Level == LogLevel.Warning && e.Message.Contains("LDAP authentication failed") && e.Exception != null) + .ShouldBeTrue("a Warning naming the LDAP failure must be logged"); + } + + /// + /// R03:2.4 — a token from /auth/token authenticates a request carrying it as a bearer: + /// the fallback-protected root AND a role-gated endpoint (the bearer principal's "Role" claims + /// resolve through IsInRole). No cookie is sent. + /// + [Fact] + public async Task Bearer_token_authenticates_api_requests_without_a_cookie() + { + var token = await LoginAndIssueTokenAsync(); + var client = NewClient(); + + var root = new HttpRequestMessage(HttpMethod.Get, "/"); + root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + (await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + var gated = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + gated.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + var gatedResponse = await client.SendAsync(gated, Ct); + gatedResponse.StatusCode.ShouldBe(HttpStatusCode.OK); + (await gatedResponse.Content.ReadAsStringAsync(Ct)).ShouldContain("viewer"); + } + + /// A tampered or garbage bearer fails closed (401), it never falls through to anonymous. + [Theory] + [InlineData("not-a-jwt")] + [InlineData("tampered")] + public async Task Bad_bearer_token_is_rejected(string kind) + { + var token = kind == "tampered" + ? (await LoginAndIssueTokenAsync()) + "x" + : kind; + var client = NewClient(); + var req = new HttpRequestMessage(HttpMethod.Get, "/"); + req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + req.Headers.Add("X-Requested-With", "XMLHttpRequest"); + + var response = await client.SendAsync(req, Ct); + + response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized); + response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); + } + + /// + /// Fix round 1 / I1 — the first-party JwtBearer handler pins HS256: an unsigned (alg: none) + /// token carrying the right issuer/audience/roles, and a token signed with a different HMAC + /// variant, both fail CLOSED with a 401 — never a 500, never a pass. + /// + [Theory] + [InlineData("none")] + [InlineData("HS384")] + public async Task Bearer_with_a_non_HS256_algorithm_is_rejected_with_401(string alg) + { + var token = alg == "none" ? UnsignedToken() : SignedToken(SecurityAlgorithms.HmacSha384); + var client = NewClient(); + var req = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + + var response = await client.SendAsync(req, Ct); + + response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized); + response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); + + // Positive control: the same claims signed HS256 with the configured key ARE accepted, so the + // 401 above measured the algorithm and not a broken token shape. + var good = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + good.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue( + "Bearer", SignedToken(SecurityAlgorithms.HmacSha256)); + (await client.SendAsync(good, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + } + + /// + /// Fix round 1 / I2 — /auth/token mints only for a COOKIE session. A bearer token used to + /// be able to mint a fresh token for itself, so a stolen token could renew itself forever and + /// outlive both its own expiry and the user's LDAP membership. Positive control: the cookie + /// session that minted the bearer still gets a token. + /// + [Fact] + public async Task Auth_token_refuses_a_bearer_only_caller_and_serves_a_cookie_session() + { + var client = NewClient(); + var loginResponse = await client.PostAsJsonAsync("/auth/login", + new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + loginResponse.EnsureSuccessStatusCode(); + + var cookieReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + AttachCookies(cookieReq, loginResponse); + var cookieResp = await client.SendAsync(cookieReq, Ct); + cookieResp.StatusCode.ShouldBe(HttpStatusCode.OK); + var bearer = (await cookieResp.Content.ReadFromJsonAsync(Ct)).GetProperty("token").GetString()!; + + // The bearer authenticates elsewhere (positive control on the token itself) … + var root = new HttpRequestMessage(HttpMethod.Get, "/"); + root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer); + (await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + // … but cannot renew itself. + var renew = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + renew.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer); + renew.Headers.Add("X-Requested-With", "XMLHttpRequest"); + var renewResp = await client.SendAsync(renew, Ct); + renewResp.StatusCode.ShouldBeOneOf(HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden); + (await renewResp.Content.ReadAsStringAsync(Ct)).ShouldNotContain("token"); + } + + private const string TestSigningKey = "test-signing-key-with-at-least-32-bytes-of-utf8-content"; + + private static IEnumerable ViewerClaims() => + [ + new(JwtTokenService.UsernameClaimType, "mallory"), + new(JwtTokenService.RoleClaimType, "Viewer"), + ]; + + /// A token signed with using the configured key bytes. + private static string SignedToken(string algorithm) + { + var keyBytes = System.Text.Encoding.UTF8.GetBytes(TestSigningKey); + // The SAME key bytes for every algorithm, so a rejection measures the algorithm pin alone. + var token = new System.IdentityModel.Tokens.Jwt.JwtSecurityToken( + issuer: "otopcua-test", audience: "otopcua-test", claims: ViewerClaims(), + expires: DateTime.UtcNow.AddMinutes(5), + signingCredentials: new SigningCredentials(new SymmetricSecurityKey(keyBytes), algorithm)); + return new System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler().WriteToken(token); + } + + /// An alg: none token with otherwise valid issuer/audience/lifetime/roles. + private static string UnsignedToken() + { + static string B64(string json) => Microsoft.IdentityModel.Tokens.Base64UrlEncoder.Encode(json); + var exp = DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeSeconds(); + return B64("{\"alg\":\"none\",\"typ\":\"JWT\"}") + "." + + B64($"{{\"zb:username\":\"mallory\",\"Role\":\"Viewer\",\"iss\":\"otopcua-test\",\"aud\":\"otopcua-test\",\"exp\":{exp}}}") + + "."; + } + + private async Task LoginAndIssueTokenAsync() + { + var client = NewClient(); + var loginResponse = await client.PostAsJsonAsync("/auth/login", + new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + loginResponse.EnsureSuccessStatusCode(); + var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + AttachCookies(tokenReq, loginResponse); + var response = await client.SendAsync(tokenReq, Ct); + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadFromJsonAsync(Ct); + return payload.GetProperty("token").GetString()!; } /// Tests that ping anonymous returns 401. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs new file mode 100644 index 00000000..a79ab569 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs @@ -0,0 +1,25 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests; + +/// Collects every log entry written through the host's logger factory, for assertions on what was logged. +internal sealed class CapturingLoggerProvider : ILoggerProvider +{ + public ConcurrentQueue Entries { get; } = new(); + + public ILogger CreateLogger(string categoryName) => new Logger(categoryName, Entries); + + public void Dispose() { } + + internal sealed record Entry(string Category, LogLevel Level, string Message, Exception? Exception); + + private sealed class Logger(string category, ConcurrentQueue entries) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + entries.Enqueue(new Entry(category, logLevel, formatter(state, exception), exception)); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs new file mode 100644 index 00000000..9d2cc7ef --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs @@ -0,0 +1,127 @@ +using System.Net; +using System.Net.Http.Json; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Services; +using ZB.MOM.WW.OtOpcUa.Security.Auth; +using ZB.MOM.WW.OtOpcUa.Security.Endpoints; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; +using ZB.MOM.WW.OtOpcUa.Security.Ldap; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests; + +/// +/// R03:2.4 — JWT is opt-in by signing key. Unconfigured: /auth/token is not mapped (it used to +/// answer HTTP 500 from the service constructor) and no bearer scheme exists. Configured badly: the +/// host refuses to START rather than failing on the first token request. +/// +public sealed class JwtConfigurationGateTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Without_a_signing_key_auth_token_is_not_mapped_and_bearer_is_ignored() + { + using var host = await StartHostAsync(signingKey: null); + var server = host.GetTestServer(); + var client = server.CreateClient(); + + // The login still works — the cookie path is untouched. + var login = await client.PostAsJsonAsync("/auth/login", new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + login.StatusCode.ShouldBe(HttpStatusCode.NoContent); + + // /auth/token is simply absent (404), not a 500. + var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + foreach (var c in login.Headers.GetValues("Set-Cookie")) tokenReq.Headers.Add("Cookie", c.Split(';')[0]); + (await client.SendAsync(tokenReq, Ct)).StatusCode.ShouldBe(HttpStatusCode.NotFound); + + // No bearer scheme is registered, so a bearer header is ignored and the cookie session still wins. + var schemes = host.Services.GetRequiredService(); + (await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer)).ShouldBeNull(); + var withBogusBearer = new HttpRequestMessage(HttpMethod.Get, "/"); + foreach (var c in login.Headers.GetValues("Set-Cookie")) withBogusBearer.Headers.Add("Cookie", c.Split(';')[0]); + withBogusBearer.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "garbage"); + (await client.SendAsync(withBogusBearer, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + await host.StopAsync(Ct); + } + + /// Positive control for the gate: with a key, the bearer scheme IS registered. + [Fact] + public async Task With_a_signing_key_the_bearer_scheme_is_registered() + { + using var host = await StartHostAsync(signingKey: "test-signing-key-with-at-least-32-bytes-of-utf8-content"); + var schemes = host.Services.GetRequiredService(); + var bearer = await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer); + bearer.ShouldNotBeNull(); + bearer.HandlerType.ShouldBe(typeof(Microsoft.AspNetCore.Authentication.JwtBearer.JwtBearerHandler)); + await host.StopAsync(Ct); + } + + /// A configured-but-short key fails at host start (ValidateOnStart), not on the first /auth/token. + [Fact] + public async Task A_short_signing_key_fails_host_start() + { + var ex = await Should.ThrowAsync(async () => + { + using var host = await StartHostAsync(signingKey: "too-short"); + }); + ex.Message.ShouldContain("SigningKey"); + } + + private static async Task StartHostAsync(string? signingKey) + { + var dbName = $"jwt-gate-{Guid.NewGuid():N}"; + var settings = new Dictionary(); + if (signingKey is not null) settings["Security:Jwt:SigningKey"] = signingKey; + var configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build(); + + var host = new HostBuilder() + .ConfigureWebHost(web => + { + web.UseTestServer(); + web.ConfigureServices(services => + { + services.AddDbContextFactory(opt => opt.UseInMemoryDatabase(dbName)); + services.AddDbContext(opt => opt.UseInMemoryDatabase(dbName)); + services.AddRouting(); + services.AddOtOpcUaAuth(configuration); + services.AddSingleton(); + services.AddSingleton(); + }); + web.Configure(app => + { + app.UseRouting(); + app.UseAuthentication(); + app.UseAuthorization(); + app.UseEndpoints(e => + { + e.MapOtOpcUaAuth(); + e.MapGet("/", () => Results.Ok("authenticated")).RequireAuthorization(); + }); + }); + }) + .Build(); + await host.StartAsync(Ct); + return host; + } + + private sealed class AlwaysAliceLdap : ILdapAuthService + { + public Task AuthenticateAsync(string username, string password, CancellationToken ct = default) => + Task.FromResult(new LdapAuthResult( + Success: password == "valid-password", DisplayName: "Alice", Username: username, + Groups: [], Roles: [], Error: null)); + } +}