diff --git a/docker-dev/README.md b/docker-dev/README.md index 6dbdbcb6..fc2e8957 100644 --- a/docker-dev/README.md +++ b/docker-dev/README.md @@ -207,7 +207,9 @@ The `-v` drops the SQL volume; remove it to keep ConfigDb state across restarts. The full single-mesh stack (`central-1`/`central-2` + the four site nodes) can OOM-kill `central-1` on a loaded host. Two settings in the compose file guard against that: -- **EF Core + ASP.NET Core logs are pinned to `Warning`** on every host node (`Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore` / `…Microsoft.AspNetCore` = `Warning`). The host logs via Serilog (`AddZbSerilog` → `ReadFrom.Configuration`), and in `Development` the default level is `Debug` — without these overrides every Deployment-poll emits an `Executed DbCommand` / `SELECT … FROM [Deployment]` line, flooding the Serilog pipeline and starving the Akka cluster heartbeat thread. Application + Akka log levels are left untouched, so this only silences the per-poll SQL chatter. To temporarily restore the SQL log flood for debugging, drop those two env vars (or set them back to `Information`) on the node you're inspecting. +- **EF Core + ASP.NET Core logs are pinned to `Warning`** on every host node (`Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore` / `…Microsoft.AspNetCore` = `Warning`). The host logs via Serilog (`AddZbSerilog` → `ReadFrom.Configuration`), and a `Development` environment would default to `Debug` (`appsettings.Development.json`) — the rig does NOT set `ASPNETCORE_ENVIRONMENT`, so nodes actually run at the base `Information` default, but these pins keep the SQL chatter out should the rig ever run as Development; without them a Debug/Information-level EF pipeline emits an `Executed DbCommand` / `SELECT … FROM [Deployment]` line, flooding the Serilog pipeline and starving the Akka cluster heartbeat thread. Application + Akka log levels are left untouched, so this only silences the per-poll SQL chatter. To temporarily restore the SQL log flood for debugging, drop those two env vars (or set them back to `Information`) on the node you're inspecting. + + To raise Debug logging for chosen categories on `central-1`/`central-2` (e.g. `ZB.MOM.WW.OtOpcUa.Runtime`), use the overlay: `docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.debug.yml up -d --no-deps central-1 central-2` — it sets `Serilog__MinimumLevel__Override__=Debug`; edit its category list to taste. - **Each host node has `mem_limit: 1g`** (`mem_reservation: 512m`). A quiet solo `central-1` measures ~357 MiB; the limit leaves headroom for the deploy/UI load and per-cluster driver subscriptions that push a fully-loaded node higher. The limit/reservation live on the `&otopcua-host` anchor, so all six host services inherit them; `sql`, `traefik`, and the one-shot `migrator`/`cluster-seed` are left unbounded. The full six-node host stack therefore needs roughly **6 GiB** of Docker Desktop VM memory just for the host nodes (plus SQL Server's own footprint on top). On a constrained host, either raise the Docker Desktop VM memory or run fewer host services (e.g. just `central-1` + `central-2`, or a single central node) rather than the full mesh. diff --git a/docker-dev/docker-compose.debug.yml b/docker-dev/docker-compose.debug.yml new file mode 100644 index 00000000..6f196d80 --- /dev/null +++ b/docker-dev/docker-compose.debug.yml @@ -0,0 +1,26 @@ +# docker-dev/docker-compose.debug.yml — optional overlay: raise Serilog levels per category on the central pair. +# +# The rig nodes run with ASPNETCORE_ENVIRONMENT unset (Production), so there is NO appsettings.Development.json and the +# Serilog default level is Information (the base appsettings.json sets none). Debug is therefore OFF by default; this +# overlay turns it on for chosen namespaces only, via Serilog:MinimumLevel:Override: (env: double underscore +# for ':', and the namespace's own dots stay dots). +# +# Up (recreates only the two central nodes with the extra env): +# docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.debug.yml up -d --no-deps central-1 central-2 +# Back to normal: re-run `up -d --no-deps central-1 central-2` with the base file alone. +# +# Edit the list below to the categories you care about. Useful ones: +# ZB.MOM.WW.OtOpcUa.Runtime actors: DriverHostActor, OpcUaPublishActor, redundancy, deploys +# ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverHostActor one actor only (e.g. the #672 historization verdict logs) +# ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient a single driver +# Opc.Ua OPC Foundation stack (very chatty) +# Akka cluster/remoting internals (chatty; base is Information) +# Leave Microsoft.EntityFrameworkCore / Microsoft.AspNetCore at Warning (see README "Resource limits & dev logging"): +# a Debug SQL flood starves the Akka heartbeat thread. +services: + central-1: + environment: + Serilog__MinimumLevel__Override__ZB.MOM.WW.OtOpcUa.Runtime: "Debug" + central-2: + environment: + Serilog__MinimumLevel__Override__ZB.MOM.WW.OtOpcUa.Runtime: "Debug" diff --git a/docs/Configuration.md b/docs/Configuration.md index 6e0bb3fb..e188d552 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -22,14 +22,30 @@ Two related concerns get their own dedicated pages and are **only summarised + l The Host (`Program.cs`) loads `appsettings.json`, then overlays a **per-role** file chosen from the cluster roles: -- A single role → `appsettings.{role}.json` (e.g. `appsettings.driver.json`, `appsettings.admin.json`). -- Both roles → `appsettings.admin-driver.json` (roles joined with `-`, ordinal-sorted). +- A single plain role → `appsettings.{role}.json` (e.g. `appsettings.driver.json`, `appsettings.admin.json`). +- Both roles → `appsettings.admin-driver.json` (plain roles joined with `-`, ordinal-sorted). +- `cluster-{ClusterId}` roles are **ignored** when naming the overlay (#678.6) — `[driver, cluster-SITE-A]` loads `appsettings.driver.json`. - `appsettings.{ASPNETCORE_ENVIRONMENT}.json` (e.g. `appsettings.Development.json`) is layered on by the host builder. All role overlays are **optional** — the base `appsettings.json` plus the Options-class C# defaults are enough to boot. The roles themselves come from `EffectiveRoles.Resolve` (`ZB.MOM.WW.OtOpcUa.Cluster`) — `Cluster:Roles`, falling back to `OTOPCUA_ROLES` when that is empty (see [`ServiceHosting.md`](ServiceHosting.md) and the table below). The checked-in `appsettings*.json` files are deliberately thin: they carry only `Serilog` and the `Security:Ldap` overlay. Everything else (`OpcUa`, `Cluster`, `ConnectionStrings`/`ConfigDb`) binds from the Options-class defaults documented below unless an operator adds the section explicitly or supplies the corresponding environment variable. + +#### Role overlay audit (#678.6) + +Until #678.6 the overlay never loaded on a node carrying a `cluster-*` role (the cluster role leaked into the filename). Fixing that **activates** the shipped overlays on those nodes (the docker-dev rig and any per-cluster-mesh production node), so each key was audited against base `appsettings.json` / the Options defaults. Env vars and command-line args are re-appended after the overlay, so any value a deployment sets explicitly (the rig sets every `Security__Ldap__*` key) still wins. + +| Key | Overlay(s) | Base / default | Effect on activation | +|---|---|---|---| +| `Serilog:MinimumLevel:Default` = `Information` | all three | Serilog default `Information` | no-op (in `Development` it outranks `appsettings.Development.json`'s `Debug`, by design) | +| `…Override:Akka` = `Information` | all three | inherits `Default` (`Information`) | no-op | +| `…Override:Microsoft.AspNetCore` = `Warning` | `admin`, `admin-driver` | unset (`Information`) | quieter ASP.NET Core logs; log-level only, matches what the rig already pins | +| `…Override:Opc.Ua` = `Information` | `admin-driver`, `driver` | inherits `Default` | no-op. **`driver` shipped `Debug` — changed to `Information` in #678.6**: activating a Debug OPC UA stack trace on every production driver node would flood Serilog (the same pipeline starvation the rig's EF/AspNetCore pins guard against). Raise it per node with `Serilog__MinimumLevel__Override__Opc.Ua=Debug`. | +| `Security:Ldap:DevStubMode` = `false` | all three | `false` | no-op; the overlay's point is that it outranks a developer's `appsettings.Development.json` (`true`) | +| `Security:Ldap:Transport` = `Ldaps` | all three | `LdapOptions.Transport` default `None` | **behaviour-changing, kept deliberately** (documented secure production default). A cluster-role node that configures `Security:Ldap:Server`/`Port` for a plaintext LDAP but never sets `Transport` would now dial LDAPS and fail AdminUI sign-in; set `Security__Ldap__Transport` (the rig already sets `None` explicitly). Data-plane auth is API-key based and unaffected; driver-only nodes have no AdminUI. | + + --- ## `appsettings` sections @@ -38,7 +54,7 @@ The checked-in `appsettings*.json` files are deliberately thin: they carry only - **Purpose:** logging. Console + rolling daily file sink, layered with the shared `ZB.MOM.WW.Telemetry` enrichers (`AddZbSerilog` in `Program.cs`). - **Where bound:** `builder.AddZbSerilog(...)` reads `Serilog` from configuration (`ReadFrom.Configuration`). -- **Checked-in shape** (`appsettings.json`): `Using` = `[ "Serilog.Sinks.Console", "Serilog.Sinks.File" ]`, `WriteTo` = a `Console` sink and a `File` sink (`path: logs/otopcua-.log`, `rollingInterval: Day`). Role overlays add `MinimumLevel` / `Override` blocks (e.g. `Opc.Ua: Debug`, `Akka: Information`). +- **Checked-in shape** (`appsettings.json`): `Using` = `[ "Serilog.Sinks.Console", "Serilog.Sinks.File" ]`, `WriteTo` = a `Console` sink and a `File` sink (`path: logs/otopcua-.log`, `rollingInterval: Day`). Role overlays add `MinimumLevel` / `Override` blocks (e.g. `Opc.Ua: Information`, `Akka: Information`). ### `OpcUa` diff --git a/docs/ServiceHosting.md b/docs/ServiceHosting.md index e5e8aa42..48b13b9b 100644 --- a/docs/ServiceHosting.md +++ b/docs/ServiceHosting.md @@ -32,13 +32,13 @@ Single-node dev: `OTOPCUA_ROLES=admin,driver` (or the `Cluster:Roles` equivalent ### Per-role configuration overlays -`Program.cs` builds a role suffix by joining the resolved `EffectiveRoles.Gossiped` roles **alphabetically** with `-` and loads `appsettings.{roleSuffix}.json` as an optional overlay on top of base `appsettings.json`. A plain role (`admin`/`driver`/`dev`) is always lower-case in `Gossiped` regardless of how the operator spelled it, so the overlay filename for those is stable (`appsettings.driver.json`); if a `cluster-{ClusterId}` role is also configured, its suffix carries into the filename **verbatim** (e.g. `Cluster:Roles: ["driver", "cluster-SITE-A"]` loads `appsettings.cluster-SITE-A-driver.json`), since `Gossiped` preserves that spelling by design (see [Configuration.md § `Roles`](Configuration.md#appsettings-sections)). Three overlays ship in `src/Server/ZB.MOM.WW.OtOpcUa.Host/`: +`Program.cs` builds a role suffix (`RoleOverlay.Suffix`) by joining the resolved `EffectiveRoles.Gossiped` **plain** roles (`admin`/`driver`/`dev`) alphabetically with `-` and loads `appsettings.{roleSuffix}.json` as an optional overlay on top of base `appsettings.json`. A plain role is always lower-case in `Gossiped` regardless of how the operator spelled it, so the overlay filename is stable (`appsettings.driver.json`). **`cluster-{ClusterId}` roles are NOT part of the filename** (#678.6): they are mesh-membership tags, so `Cluster:Roles: ["driver", "cluster-SITE-A"]` loads `appsettings.driver.json` and `["admin", "driver", "cluster-MAIN"]` loads `appsettings.admin-driver.json`. (Before #678.6 the cluster role was folded into the name, producing e.g. `appsettings.admin-cluster-MAIN-driver.json`, which never exists — so on every node with a cluster role no shipped overlay ever loaded.) Three overlays ship in `src/Server/ZB.MOM.WW.OtOpcUa.Host/`: - `appsettings.admin.json` — admin-only nodes - `appsettings.driver.json` — driver-only nodes - `appsettings.admin-driver.json` — fused single-node dev / small deployments -All three carry Serilog log-level overrides + `Security:Ldap:DevStubMode = false`. Configuration loading order (lowest to highest precedence) is: +All three carry Serilog log-level overrides + `Security:Ldap:DevStubMode = false` + `Security:Ldap:Transport = Ldaps` (audit table: [Configuration.md § Role overlays](Configuration.md#role-overlay-audit)). Configuration loading order (lowest to highest precedence) is: `appsettings.json` < `appsettings.{Environment}.json` < `appsettings.{role}.json` < environment variables < command-line args diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbSchemaBootstrapper.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbSchemaBootstrapper.cs index 296f787d..f610fac5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbSchemaBootstrapper.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbSchemaBootstrapper.cs @@ -137,7 +137,10 @@ internal sealed class QuestDbSchemaBootstrapper // The ALTER is a WAL transaction: tables() keeps the old value until it applies, so do NOT re-read to verify. await using (var alter = _conn.CreateCommand(conn, QuestDbTables.AlterTtl(table, _options.RetentionDays))) await alter.ExecuteNonQueryAsync(ct).ConfigureAwait(false); - _log.LogInformation("Applied TTL {Days} day(s) to QuestDB table {Table}.", _options.RetentionDays, table); + if (_options.RetentionDays is { } days) + _log.LogInformation("Applied TTL {Days} day(s) to QuestDB table {Table}.", days, table); + else + _log.LogInformation("Cleared TTL on QuestDB table {Table}.", table); } } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianRetention.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianRetention.cs index c8bc8e00..24816415 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianRetention.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianRetention.cs @@ -42,6 +42,9 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention /// Test seam for . internal Action ClearPool { get; set; } = SqlConnection.ClearPool; + /// Test seam: the release statement. Tests point it at a failing statement to exercise the pool-clear fallback. + internal string ReleaseSql { get; set; } = "EXEC sp_releaseapplock @Resource = @res, @LockOwner = 'Session';"; + public async Task> PurgeAsync(CancellationToken ct) { await _bootstrap.EnsureAsync(ct).ConfigureAwait(false); @@ -137,8 +140,7 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention { try { - await using var cmd = _conn.CreateCommand(conn, - "EXEC sp_releaseapplock @Resource = @res, @LockOwner = 'Session';"); + await using var cmd = _conn.CreateCommand(conn, ReleaseSql); cmd.Parameters.Add(new SqlParameter("@res", SqlDbType.NVarChar, 255) { Value = LockResource }); await cmd.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/RoleOverlay.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/RoleOverlay.cs new file mode 100644 index 00000000..b7ce2047 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/RoleOverlay.cs @@ -0,0 +1,28 @@ +using ZB.MOM.WW.OtOpcUa.Cluster; + +namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; + +/// +/// Names the per-role appsettings overlay (appsettings.{suffix}.json) for a node. +/// +/// +/// The suffix is built from the node's PLAIN roles only. cluster-{ClusterId} roles are mesh-membership +/// tags (and carry a verbatim, operator-chosen ClusterId), not configuration profiles — folding one into the +/// filename produced names like appsettings.admin-cluster-MAIN-driver.json that never exist, so on every +/// node with a cluster role the shipped overlays silently never loaded. +/// +internal static class RoleOverlay +{ + /// + /// The overlay filename suffix: the plain (non-cluster) roles, ordinal-sorted and joined with -, or + /// when the node has none. + /// + public static string? Suffix(IReadOnlyList gossipedRoles) + { + var plain = gossipedRoles + .Where(r => !string.IsNullOrWhiteSpace(r) && !RoleParser.IsClusterRole(r)) + .OrderBy(r => r, StringComparer.Ordinal) + .ToList(); + return plain.Count == 0 ? null : string.Join('-', plain); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 29dd14a3..ee5e79d7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -90,8 +90,9 @@ builder.WebHost.UseStaticWebAssets(); builder.Host.UseWindowsService(options => options.ServiceName = "OtOpcUaHost"); // Per-role appsettings overlay: appsettings.{role}.json (single role) or appsettings.admin-driver.json -// (both). Optional — base appsettings.json carries enough to boot if these don't exist. -var roleSuffix = roles.Count == 0 ? null : string.Join('-', roles.OrderBy(r => r, StringComparer.Ordinal)); +// (both). Optional — base appsettings.json carries enough to boot if these don't exist. Built from the +// PLAIN roles only (#678.6): a cluster-{ClusterId} role in the name pointed at a file that never exists. +var roleSuffix = RoleOverlay.Suffix(roles); if (roleSuffix is not null) { builder.Configuration.AddJsonFile($"appsettings.{roleSuffix}.json", optional: true, reloadOnChange: true); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/appsettings.driver.json b/src/Server/ZB.MOM.WW.OtOpcUa.Host/appsettings.driver.json index 1db8bd01..89cf7f6f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/appsettings.driver.json +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/appsettings.driver.json @@ -3,7 +3,7 @@ "MinimumLevel": { "Default": "Information", "Override": { - "Opc.Ua": "Debug", + "Opc.Ua": "Information", "Akka": "Information" } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendTests.cs index 19ef82d0..6b2f619e 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendTests.cs @@ -71,7 +71,7 @@ public sealed class WireBackendTests await driver.InitializeAsync("{}", ct); await WaitFor(() => - driver.GetDeviceState(DeviceHost) is { FixedTreeCache: not null }, TimeSpan.FromSeconds(5)); + driver.GetDeviceState(DeviceHost) is { FixedTreeCache: not null }, WaitTimeout); var state = driver.GetDeviceState(DeviceHost); state.ShouldNotBeNull(); @@ -82,7 +82,7 @@ public sealed class WireBackendTests await WaitFor(() => state.LastFixedSnapshots.ContainsKey($"{DeviceHost}/Axes/X/AbsolutePosition"), - TimeSpan.FromSeconds(3)); + WaitTimeout); state.LastFixedSnapshots[$"{DeviceHost}/Axes/X/AbsolutePosition"].ShouldBe(123456); } } @@ -136,7 +136,7 @@ public sealed class WireBackendTests await WaitFor(() => driver.GetDeviceState(DeviceHost) is { LastProgramInfo: not null }, - TimeSpan.FromSeconds(5)); + WaitTimeout); var snapshots = await driver.ReadAsync( [$"{DeviceHost}/Program/Name", @@ -194,7 +194,7 @@ public sealed class WireBackendTests { var state = driver.GetDeviceState(DeviceHost); return state is not null && state.LastTimers.Count == 4; - }, TimeSpan.FromSeconds(5)); + }, WaitTimeout); var snapshots = await driver.ReadAsync( [$"{DeviceHost}/Timers/PowerOnSeconds", @@ -265,8 +265,11 @@ public sealed class WireBackendTests { var state = driver.GetDeviceState(DeviceHost); return state?.FixedTreeCache is { Capabilities.SpindleLoad: true, Capabilities.SpindleMotorSpeed: true } - && state.LastSpindleLoads.Count >= 2; - }, TimeSpan.FromSeconds(5)); + && state.LastSpindleLoads.Count >= 2 + // Motor speeds are cached after a SECOND wire round-trip (FocasDriver spindle poll), so the loads + // arriving first does not mean the speeds have — waiting on loads alone raced under load (#678.3). + && state.LastSpindleMotorSpeeds.Count >= 2; + }, WaitTimeout); var snapshots = await driver.ReadAsync( [$"{DeviceHost}/Spindle/S1/Load", @@ -282,6 +285,10 @@ public sealed class WireBackendTests } } + /// Generous: the sim fixture answers fast, but a loaded CI host (e.g. a concurrent docker build) does not. + private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(15); + + /// Polls ; FAILS the test with a clear message on timeout rather than returning silently. private static async Task WaitFor(Func pred, TimeSpan timeout) { var deadline = DateTime.UtcNow + timeout; @@ -290,5 +297,7 @@ public sealed class WireBackendTests if (pred()) return; await Task.Delay(50); } + if (pred()) return; + Assert.Fail($"Condition not met within {timeout.TotalSeconds:0.#}s waiting on the FOCAS wire backend."); } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/RetentionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/RetentionTests.cs index 046869d6..50870b4a 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/RetentionTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/RetentionTests.cs @@ -178,6 +178,80 @@ public sealed class RetentionTests(SqlHistorianFixture fx) (await LockIsFreeAsync(s)).ShouldBeTrue(); } + [Fact] + public async Task Cancelling_mid_command_while_the_delete_is_blocked_still_releases_the_applock() + { + fx.SkipIfUnavailable(); + var (c, b, _, s) = Make(); + await SeedAsync(b, s); + + // A second session holds an exclusive table lock, so the purger's first DELETE blocks server-side. + await using var holder = new SqlConnection(fx.ConnectionString); + await holder.OpenAsync(); + await using var tx = (SqlTransaction)await holder.BeginTransactionAsync(); + await using (var cmd = holder.CreateCommand()) + { + cmd.Transaction = tx; + cmd.CommandText = $"SELECT COUNT(*) FROM [{s}].Sample WITH (TABLOCKX, HOLDLOCK)"; + await cmd.ExecuteScalarAsync(); + } + + using var cts = new CancellationTokenSource(); + var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now)); + var purge = purger.PurgeAsync(cts.Token); + await Task.Delay(500); + purge.IsCompleted.ShouldBeFalse("the DELETE must be blocked behind the holder's table lock"); + cts.Cancel(); + + // SqlClient may surface the attention as a SqlException rather than OperationCanceledException. + await Should.ThrowAsync(() => purge); + + await tx.RollbackAsync(); + (await LockIsFreeAsync(s)).ShouldBeTrue(); + (await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(21); + } + + [Fact] + public async Task A_failing_release_falls_back_to_clearing_the_pool_so_the_applock_is_freed() + { + fx.SkipIfUnavailable(); + var (c, b, _, s) = Make(); + await SeedAsync(b, s); + var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now)) + { + ReleaseSql = "EXEC sp_this_procedure_does_not_exist;", + }; + + await purger.PurgeAsync(CancellationToken.None); + + (await LockIsFreeAsync(s)).ShouldBeTrue(); + } + + [Fact] + public async Task Falsification_control_a_failing_release_with_a_no_op_ClearPool_leaves_the_applock_held() + { + fx.SkipIfUnavailable(); + var (c, b, _, s) = Make(); + await SeedAsync(b, s); + var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now)) + { + ReleaseSql = "EXEC sp_this_procedure_does_not_exist;", + ClearPool = _ => { }, + }; + + await purger.PurgeAsync(CancellationToken.None); + + try + { + // The pooled session still owns the Session-scoped lock: proves the fallback test above is not vacuous. + (await LockIsFreeAsync(s)).ShouldBeFalse(); + } + finally + { + SqlConnection.ClearAllPools(); // drop the leaked session so the lock does not outlive the test + } + } + [Fact] public async Task A_throwing_purge_releases_the_applock() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/RoleOverlayTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/RoleOverlayTests.cs new file mode 100644 index 00000000..80622a4d --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/RoleOverlayTests.cs @@ -0,0 +1,78 @@ +using System.Text.Json; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration; + +/// +/// #678.6 — the per-role appsettings overlay name is built from plain roles only. A cluster-* role in the +/// name produced a file that never exists, so no overlay ever loaded on a node with a cluster role. +/// +public sealed class RoleOverlayTests +{ + [Fact] + public void A_driver_site_node_loads_the_driver_overlay() + => RoleOverlay.Suffix(["driver", "cluster-SITE-A"]).ShouldBe("driver"); + + [Fact] + public void A_central_node_loads_the_admin_driver_overlay() + => RoleOverlay.Suffix(["admin", "driver", "cluster-MAIN"]).ShouldBe("admin-driver"); + + [Fact] + public void Plain_roles_are_ordinal_sorted_regardless_of_input_order() + => RoleOverlay.Suffix(["driver", "cluster-MAIN", "admin"]).ShouldBe("admin-driver"); + + [Fact] + public void Roles_without_a_cluster_role_keep_the_existing_names() + { + RoleOverlay.Suffix(["admin"]).ShouldBe("admin"); + RoleOverlay.Suffix(["driver"]).ShouldBe("driver"); + RoleOverlay.Suffix(["admin", "driver"]).ShouldBe("admin-driver"); + } + + [Fact] + public void A_cluster_role_prefix_is_matched_case_insensitively() + => RoleOverlay.Suffix(["driver", "CLUSTER-SITE-A"]).ShouldBe("driver"); + + [Fact] + public void No_plain_role_means_no_overlay() + { + RoleOverlay.Suffix([]).ShouldBeNull(); + RoleOverlay.Suffix(["cluster-SITE-A"]).ShouldBeNull(); + } + + // ---- The shipped overlays: activating them on cluster-role nodes must not change production semantics. ---- + + private static string HostDir() + { + for (var dir = new DirectoryInfo(AppContext.BaseDirectory); dir is not null; dir = dir.Parent) + { + var candidate = Path.Combine(dir.FullName, "src", "Server", "ZB.MOM.WW.OtOpcUa.Host"); + if (File.Exists(Path.Combine(candidate, "appsettings.json"))) return candidate; + } + + throw new InvalidOperationException("Host project directory not found above " + AppContext.BaseDirectory); + } + + [Theory] + [InlineData("admin")] + [InlineData("driver")] + [InlineData("admin-driver")] + public void Shipped_overlays_carry_only_the_audited_keys(string suffix) + { + using var doc = JsonDocument.Parse(File.ReadAllText(Path.Combine(HostDir(), $"appsettings.{suffix}.json"))); + var root = doc.RootElement; + + // Only Serilog + Security:Ldap, and in Security:Ldap only the two audited keys. + root.EnumerateObject().Select(p => p.Name).Order().ShouldBe(["Security", "Serilog"]); + var ldap = root.GetProperty("Security").GetProperty("Ldap"); + ldap.EnumerateObject().Select(p => p.Name).Order().ShouldBe(["DevStubMode", "Transport"]); + ldap.GetProperty("DevStubMode").GetBoolean().ShouldBeFalse(); + + // No Debug-level (or lower) category override: activating an overlay must not raise log volume on a node. + foreach (var o in root.GetProperty("Serilog").GetProperty("MinimumLevel").GetProperty("Override").EnumerateObject()) + o.Value.GetString().ShouldNotBeOneOf("Debug", "Verbose"); + root.GetProperty("Serilog").GetProperty("MinimumLevel").GetProperty("Default").GetString().ShouldBe("Information"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorHistorizationRoleViewTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorHistorizationRoleViewTests.cs index 0ca0dcac..dee3ccec 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorHistorizationRoleViewTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorHistorizationRoleViewTests.cs @@ -223,10 +223,30 @@ public sealed class DriverHostActorHistorizationRoleViewTests : RuntimeActorTest host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672))); AwaitAssert(() => hist.Published.ShouldBe([false, true]), duration: Timeout); - // And self "returning" does not close what the peer's loss opened. + // Self "returning" is ignored too, and it does not close what the peer's loss opened. Events are + // processed in order, so the peer's return (which MUST close the gate) is the only thing that may + // appear next: had the self event published anything it would sit between `true` and `false`. host.Tell(new ClusterEvent.ReachableMember(MemberAt(Self672))); + host.Tell(new ClusterEvent.ReachableMember(MemberAt(Peer672))); + AwaitAssert(() => hist.Published.ShouldBe([false, true, false]), duration: Timeout); + } + + /// + /// A host that starts while its peer is ALREADY unreachable (the InitialStateAsEvents replay + /// delivers the UnreachableMember before the first redundancy snapshot) must still treat that + /// peer as unreachable when the snapshot naming it Primary finally arrives. + /// + [Fact] + public void A_peer_unreachable_before_the_first_snapshot_still_opens_capture() + { + var hist = new RecordingHistorizationView(); + IActorRef host = SpawnHost672(hist, others: null, captureWhilePrimaryUnreachable: true); + + host.Tell(new ClusterEvent.UnreachableMember(MemberAt(Peer672))); host.Tell(Snapshot((Self672, RedundancyRole.Secondary), (Peer672, RedundancyRole.Primary))); - AwaitAssert(() => hist.Published.ShouldBe([false, true, true]), duration: Timeout); + + // Exactly one publication, and it is `true`: no spurious `false` from treating the peer as reachable. + AwaitAssert(() => hist.Published.ShouldBe([true]), duration: Timeout); } /// CaptureWhilePrimaryUnreachable = false restores the pre-#672 behaviour.