diff --git a/CLAUDE.md b/CLAUDE.md index 06c1f7e5..2e2db504 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -219,10 +219,13 @@ The server supports configurable OPC UA transport security via the `OpcUa:Enable The server supports non-transparent warm/hot redundancy via the `Redundancy` section in `appsettings.json`. Two instances share the same Galaxy DB and the same mxaccessgw (under distinct `MxAccess.ClientName` values) but have unique `ApplicationUri` values. Each exposes `RedundancySupport`, `ServerUriArray`, and a dynamic `ServiceLevel` based on role and runtime health. The primary advertises a higher ServiceLevel than the secondary. See `docs/Redundancy.md` for the full guide. -## LocalDb pair-local config cache (Phase 1) +## LocalDb pair-local store (Phases 1 + 2) Every **driver-role** node keeps a consolidated `ZB.MOM.WW.LocalDb` SQLite database (the retired -LiteDB `LocalCache` subsystem was deleted as superseded). Phase-1 scope: it caches the +LiteDB `LocalCache` subsystem was deleted as superseded). It holds **three replicated tables**: +`deployment_artifacts` + `deployment_pointer` (Phase 1) and `alarm_sf_events` (Phase 2). + +**Phase 1** caches the **deployed-configuration artifact** (chunked, SHA-256-verified, newest-2 retention per cluster) so a driver node can **boot from cache when central SQL Server is unreachable** — `DriverHostActor` writes the cache after each successful apply (`IDeploymentArtifactCache` → `LocalDbDeploymentArtifactCache`, @@ -234,8 +237,27 @@ library's gRPC sync — **default-OFF, fail-closed bearer auth** (`LocalDbSyncAu on `LocalDb:SyncListenPort` (a dedicated Kestrel h2c listener) + `LocalDb:Replication:*`. On the docker-dev rig the **site-a pair** has replication on; central + site-b stay default-OFF (site-b is the pin). ApiKey via `${secret:}`/env, never committed (the rig's dev key is the committed-dev-secret -exception). See `docs/Configuration.md` (`LocalDb` section), `docs/Redundancy.md` (pair-local config -cache), and the runbook `docs/operations/2026-07-20-localdb-pair-replication.md`. +exception). + +**Phase 2** moved the **alarm store-and-forward buffer** off its standalone `alarm-historian.db` +and into the same database as `alarm_sf_events`, so a node that dies holding undelivered alarm +history no longer takes it to the grave. `SqliteStoreAndForwardSink` became +`LocalDbStoreAndForwardSink` (`Core.AlarmHistorian`), and the breaking config key +`AlarmHistorian:DatabasePath` was **removed** — it is still read, by `AlarmSfLegacyMigrator`, purely +to find a pre-consolidation file to copy across on first boot (renamed `.migrated` after the copy +commits). Because the buffer replicates, **only the node holding the Primary role drains it**: the +sink takes a `drainGate` delegate, Runtime supplies it from the `IRedundancyRoleView` singleton that +`DriverHostActor` publishes its `PrimaryGatePolicy` verdict to, and a gated-off node reports the new +`HistorianDrainState.NotPrimary`. Delivery is therefore **at-least-once across a failover, by +design** (no dedup layer). Row ids are a SHA-256 of the event payload rather than a GUID, so the +same event accepted on both nodes — which happens in every boot window, since +`HistorianAdapterActor` default-writes while its role is unknown — converges to one row. On the +docker-dev rig the site-a pair and site-b both run `AlarmHistorian__Enabled=true` against a +deliberately unresolvable gateway endpoint, so the buffer is always a live, non-draining queue. + +See `docs/Configuration.md` (`LocalDb` section), `docs/Redundancy.md` (pair-local config +cache), `docs/AlarmHistorian.md`, and the runbook +`docs/operations/2026-07-20-localdb-pair-replication.md`. ## LDAP Authentication @@ -339,11 +361,13 @@ authorization uses the standard `AccessLevels.HistoryRead` bit set at materializ ### Alarm-history path (`AlarmHistorian` section) Alarm events are written through `GatewayAlarmHistorianWriter` (the gateway **`SendEvent`** path) behind -the durable **`SqliteStoreAndForwardSink`** — `AlarmHistorian:Enabled=true` swaps the `NullAlarmHistorianSink` -default for the SQLite store-and-forward queue, whose drain worker forwards batches to the gateway and uses -per-event outcomes to decide retry vs. dead-letter (never throws). The `AlarmHistorian` section carries -only the `Enabled` gate + the SQLite knobs (`DatabasePath`, `DrainIntervalSeconds`, `Capacity`, -`DeadLetterRetentionDays`, `BatchSize`, `MaxAttempts`) — the downstream gateway connection +the durable **`LocalDbStoreAndForwardSink`** — `AlarmHistorian:Enabled=true` swaps the `NullAlarmHistorianSink` +default for the store-and-forward queue, which buffers into the node's consolidated LocalDb as the +replicated `alarm_sf_events` table (see the LocalDb section) and whose drain worker forwards batches to +the gateway and uses per-event outcomes to decide retry vs. dead-letter (never throws). **Only the node +holding the Primary role drains** (a gated node reports `HistorianDrainState.NotPrimary`). The +`AlarmHistorian` section carries only the `Enabled` gate + the queue knobs (`DrainIntervalSeconds`, +`Capacity`, `DeadLetterRetentionDays`, `BatchSize`, `MaxAttempts`) — the downstream gateway connection (endpoint/key/TLS) is sourced from the `ServerHistorian` section. **Alarm-history `ReadEvents` requires the target gateway deployed with `RuntimeDb:EventReadsEnabled=true`** (the C2 SQL event-read workaround). diff --git a/Directory.Packages.props b/Directory.Packages.props index 82221614..9754f764 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -57,6 +57,7 @@ + diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index 2d2d712d..b6b33a7d 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -328,6 +328,31 @@ services: # site-a-1 is the initiator: it binds the h2c sync listener on 9001 AND dials the peer. # Setting SyncListenPort makes Program.cs add a dedicated Http2-only Kestrel listener and # re-bind the primary HTTP port (an explicit Listen* otherwise discards ASPNETCORE_URLS). + # Alarm store-and-forward buffer, Phase 2. Enabled on BOTH halves of the pair so the + # buffer is a live, replicated table rather than an empty one — the live gate needs a real + # queue to watch converge, and needs to see that only the Primary drains it. + # + # There is no HistorianGateway on this rig, so the endpoint below is deliberately + # unresolvable: every drain attempt fails, which is exactly the historian-outage state the + # buffer exists for. Note the endpoint must still be a syntactically valid absolute http(s) + # URI or the host refuses to start — ServerHistorianOptionsValidator is consumer-gated, so + # AlarmHistorian:Enabled=true makes it required even while ServerHistorian:Enabled=false. + # + # MaxAttempts is raised far above the production default of 10. With a permanently + # unreachable gateway the default would dead-letter the whole queue about five minutes into + # the outage, turning a buffering test into a dead-letter test. + AlarmHistorian__Enabled: "true" + AlarmHistorian__MaxAttempts: "1000000" + ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" + # Required whenever the section is consumed: the gateway client validates its own options at + # construction and throws on an empty key, so a keyless node crash-loops during Akka startup + # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint + # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via + # ServerHistorian__ApiKey from the environment. + ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" + # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER + # direction, so an http endpoint with the default UseTls=true crash-loops the host. + ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" LocalDb__SyncListenPort: "9001" LocalDb__Replication__PeerAddress: "http://site-a-2:9001" @@ -365,6 +390,31 @@ services: # but does NOT dial (no PeerAddress). The replication stream is bidirectional, so a-1's # single dial carries both directions. Same key + MaxBatchSize as a-1 (byte-identical key # is mandatory — the interceptor fail-closes on a mismatch). + # Alarm store-and-forward buffer, Phase 2. Enabled on BOTH halves of the pair so the + # buffer is a live, replicated table rather than an empty one — the live gate needs a real + # queue to watch converge, and needs to see that only the Primary drains it. + # + # There is no HistorianGateway on this rig, so the endpoint below is deliberately + # unresolvable: every drain attempt fails, which is exactly the historian-outage state the + # buffer exists for. Note the endpoint must still be a syntactically valid absolute http(s) + # URI or the host refuses to start — ServerHistorianOptionsValidator is consumer-gated, so + # AlarmHistorian:Enabled=true makes it required even while ServerHistorian:Enabled=false. + # + # MaxAttempts is raised far above the production default of 10. With a permanently + # unreachable gateway the default would dead-letter the whole queue about five minutes into + # the outage, turning a buffering test into a dead-letter test. + AlarmHistorian__Enabled: "true" + AlarmHistorian__MaxAttempts: "1000000" + ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" + # Required whenever the section is consumed: the gateway client validates its own options at + # construction and throws on an empty key, so a keyless node crash-loops during Akka startup + # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint + # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via + # ServerHistorian__ApiKey from the environment. + ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" + # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER + # direction, so an http endpoint with the default UseTls=true crash-loops the host. + ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" LocalDb__SyncListenPort: "9001" LocalDb__Replication__ApiKey: "dev-site-a-localdb-sync-key" @@ -397,6 +447,20 @@ services: # site-b is the default-OFF pin: it gets the pair-local cache but NO SyncListenPort and NO # replication config, so no sync listener binds and ISyncStatus stays disconnected/Healthy. # This is what the live gate's check 8 verifies — the cache works locally with replication off. + # Alarm store-and-forward with replication OFF — the other half of the site-b pin. The sink + # must work as a plain node-local buffer here, with no sync listener and no peer traffic. + AlarmHistorian__Enabled: "true" + AlarmHistorian__MaxAttempts: "1000000" + ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" + # Required whenever the section is consumed: the gateway client validates its own options at + # construction and throws on an empty key, so a keyless node crash-loops during Akka startup + # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint + # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via + # ServerHistorian__ApiKey from the environment. + ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" + # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER + # direction, so an http endpoint with the default UseTls=true crash-loops the host. + ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" ports: - "4844:4840" @@ -422,6 +486,20 @@ services: Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" # site-b default-OFF pin (see site-b-1): cache on, replication off. + # Alarm store-and-forward with replication OFF — the other half of the site-b pin. The sink + # must work as a plain node-local buffer here, with no sync listener and no peer traffic. + AlarmHistorian__Enabled: "true" + AlarmHistorian__MaxAttempts: "1000000" + ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" + # Required whenever the section is consumed: the gateway client validates its own options at + # construction and throws on an empty key, so a keyless node crash-loops during Akka startup + # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint + # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via + # ServerHistorian__ApiKey from the environment. + ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" + # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER + # direction, so an http endpoint with the default UseTls=true crash-loops the host. + ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" ports: - "4845:4840" diff --git a/docs/AlarmHistorian.md b/docs/AlarmHistorian.md index 212afe4c..26dc8efd 100644 --- a/docs/AlarmHistorian.md +++ b/docs/AlarmHistorian.md @@ -42,7 +42,7 @@ unless noted. - **`NullAlarmHistorianSink`** — the no-op default for tests and deployments that don't historize alarms. It is the default DI binding (registered in the Runtime's `AddOtOpcUaRuntime`); production overrides it with - `SqliteStoreAndForwardSink`. + `LocalDbStoreAndForwardSink`. - **`AlarmHistorianEvent`** ([`AlarmHistorianEvent.cs`](../src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmHistorianEvent.cs)) — the source-agnostic event record: `AlarmId`, `EquipmentPath` (UNS path, @@ -61,42 +61,84 @@ unless noted. - **`HistorianSinkStatus`** — diagnostic snapshot surfaced to the AdminUI and `/healthz`: `QueueDepth`, `DeadLetterDepth`, `LastDrainUtc`, `LastSuccessUtc`, `LastError`, `DrainState`, and `EvictedCount`. -- **`HistorianDrainState`** — `Disabled` / `Idle` / `Draining` / `BackingOff`. +- **`HistorianDrainState`** — `Disabled` / `Idle` / `Draining` / `BackingOff` / + **`NotPrimary`** (ticking, but leaving the replicated queue for the node that + holds the Primary role). --- -## SqliteStoreAndForwardSink +## LocalDbStoreAndForwardSink -[`SqliteStoreAndForwardSink.cs`](../src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/SqliteStoreAndForwardSink.cs) -is the production `IAlarmHistorianSink`. Construction takes a SQLite database -path, an `IAlarmHistorianWriter`, a logger, and optional `batchSize` (default -100), `capacity` (default 1,000,000), `deadLetterRetention` (default 30 days), -and a test clock. +[`LocalDbStoreAndForwardSink.cs`](../src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs) +is the production `IAlarmHistorianSink`. Construction takes the node's +`ILocalDb`, an `IAlarmHistorianWriter`, a logger, and optional `batchSize` +(default 100), `capacity` (default 1,000,000), `deadLetterRetention` (default 30 +days), a test clock, and a **`drainGate`**. + +### The drain gate + +`drainGate` is a `Func` consulted at the top of every tick; `false` skips +the tick entirely, leaving the queue untouched and reporting +`DrainState = NotPrimary`. + +It exists because the queue **replicates**. The Secondary holds a full copy of +the Primary's rows, and an ungated drain there would re-deliver every event, +continuously — not just at a failover. Runtime supplies the gate from +`IRedundancyRoleView`, a singleton `DriverHostActor` publishes its +`PrimaryGatePolicy` verdict to on every redundancy snapshot, so the drain agrees +with the inbound-write and native-ack gates by construction. + +Two postures are deliberate: + +- **Unset (the default) means drain.** So does an unpublished view. A deployment + that runs no redundancy never publishes a role, and defaulting closed would + silently stop its alarm history forever. +- **A gate that throws means "not now", never permission.** Draining on a gate + fault would put a pair into dual delivery exactly when its redundancy signal is + sick. ### Queue table -The sink owns one SQLite table (created on construction, WAL journal mode): +The sink owns one table in the node's consolidated LocalDb — created and +registered for replication by `LocalDbSetup.OnReady`, before any consumer can +resolve the sink: ```sql -CREATE TABLE Queue ( - RowId INTEGER PRIMARY KEY AUTOINCREMENT, - AlarmId TEXT NOT NULL, - EnqueuedUtc TEXT NOT NULL, - PayloadJson TEXT NOT NULL, -- JSON-serialized AlarmHistorianEvent - AttemptCount INTEGER NOT NULL DEFAULT 0, - LastAttemptUtc TEXT NULL, - LastError TEXT NULL, - DeadLettered INTEGER NOT NULL DEFAULT 0 +CREATE TABLE alarm_sf_events ( + id TEXT NOT NULL PRIMARY KEY, -- SHA-256 of payload_json + alarm_id TEXT NOT NULL, + enqueued_at_utc TEXT NOT NULL, + payload_json TEXT NOT NULL, -- JSON-serialized AlarmHistorianEvent + attempt_count INTEGER NOT NULL DEFAULT 0, + last_attempt_utc TEXT NULL, + last_error TEXT NULL, + dead_lettered INTEGER NOT NULL DEFAULT 0 ); -CREATE INDEX IX_Queue_Drain ON Queue (DeadLettered, RowId); +CREATE INDEX ix_alarm_sf_events_drain + ON alarm_sf_events (dead_lettered, enqueued_at_utc, id); ``` -`EnqueueAsync` does a single `INSERT` on the hot path. To avoid a -`SELECT COUNT(*)` on every enqueue, the sink keeps an in-memory non-dead-lettered -row counter (seeded at startup, kept current by every mutation, and re-synced -from storage every 10,000 enqueues to defend against drift). SQLite writer -contention is handled via `PRAGMA busy_timeout=5000` + WAL so an enqueue/drain -collision waits out the file lock instead of failing fast. +**The primary key is a hash of the payload, not an autoincrement rowid.** Two +reasons. Replication converges last-writer-wins *per primary key*, so two nodes +independently allocating rowid 7 to different alarms would silently overwrite one +another. And an equal-payload key makes the same event arriving twice collapse +into one row — which happens for real, because `HistorianAdapterActor` +default-writes while its redundancy role is unknown and both nodes of a pair +therefore accept the same fanned transition in every boot window. Two genuinely +distinct events cannot collide: `AlarmHistorianEvent` carries a full-precision +timestamp alongside the alarm id, kind, message and user. + +Ordering follows from that: the drain reads in `enqueued_at_utc` order (with `id` +as a tiebreak so the ordering is total), because a hashed key carries no +insertion sequence. + +`EnqueueAsync` does a single `INSERT … ON CONFLICT DO NOTHING` on the hot path. +To avoid a `SELECT COUNT(*)` on every enqueue, the sink keeps an in-memory +non-dead-lettered row counter (seeded at startup, kept current by every mutation, +and re-synced from storage every 10,000 enqueues to defend against drift — now +doubly warranted, since replication applies the peer's rows straight into the +table where no local code path can observe them). Pragmas and connection +lifetime belong to LocalDb; the sink no longer manages either. ### Drain worker @@ -104,13 +146,17 @@ collision waits out the file lock instead of failing fast. `System.Threading.Timer`** (not started automatically — tests drive `DrainOnceAsync` deterministically). Each tick: +0. Consults the drain gate. A closed gate returns immediately, leaving the queue + untouched and `DrainState = NotPrimary`. 1. Purges aged dead-lettered rows past the retention window. -2. Reads up to `batchSize` non-dead-lettered rows in `RowId` order. +2. Reads up to `batchSize` non-dead-lettered rows in `enqueued_at_utc, id` order. 3. Rows with un-deserializable payloads are dead-lettered immediately (by their - own `RowId`) so they can't stall the queue head. + own `id`) so they can't stall the queue head. 4. The remaining batch is handed to `IAlarmHistorianWriter.WriteBatchAsync`, and - each outcome is applied in one transaction: `Ack` deletes the row, - `PermanentFail` flips its `DeadLettered` flag, `RetryPlease` bumps its attempt + each outcome is applied in one transaction: `Ack` deletes the row (the delete + replicates as a tombstone, so the peer drops its copy and a promoted standby + cannot re-send it), `PermanentFail` flips its `dead_lettered` flag, + `RetryPlease` bumps its attempt count and leaves it queued. A row whose `AttemptCount` has reached the configured **`MaxAttempts`** cap (default 10) is dead-lettered automatically on the next drain tick rather than retried — this breaks infinite retry loops for poison events whose @@ -131,7 +177,7 @@ an unobserved task exception. **The durability guarantee is bounded by `capacity` (default 1,000,000 rows).** When the non-dead-lettered queue reaches capacity, `EnqueueAsync` evicts the -oldest non-dead-lettered rows (oldest `RowId` first) to make room, logs a WARN, +oldest non-dead-lettered rows (oldest `enqueued_at_utc` first) to make room, logs a WARN, and increments `HistorianSinkStatus.EvictedCount`. Under a sustained historian outage, accepted alarm events can therefore be dropped before delivery. A non-zero `EvictedCount` is a data-loss signal that requires operator attention — @@ -140,7 +186,7 @@ it surfaces silent loss without log scraping. ### Dead-letter + operator recovery `PermanentFail` and corrupt-payload rows are retained in-place with -`DeadLettered = 1` for the retention window (default 30 days) so operators can +`dead_lettered = 1` for the retention window (default 30 days) so operators can inspect them before the sweeper purges them. `RetryDeadLettered()` is the operator action (from the AdminUI) that clears the dead-letter flag and attempt count on every dead-lettered row, returning them to the regular queue with a @@ -173,17 +219,28 @@ the `alerts` topic (future). The real sink is opt-in via the `AlarmHistorian` section of `appsettings.json`. When `Enabled` is `false` (the default), `AddAlarmHistorian` registers `NullAlarmHistorianSink` and the feature is dormant. When `Enabled` is `true`, -`AddAlarmHistorian` constructs `SqliteStoreAndForwardSink` and registers +`AddAlarmHistorian` constructs `LocalDbStoreAndForwardSink` and registers `GatewayAlarmHistorianWriter` as the `IAlarmHistorianWriter`. This section carries -**only** the `Enabled` gate + the SQLite store-and-forward knobs — the downstream +**only** the `Enabled` gate + the store-and-forward knobs — the downstream gateway connection (endpoint / key / TLS) is sourced from the `ServerHistorian` -section (see [Historian.md](Historian.md)). +section (see [Historian.md](Historian.md)), and the queue's storage location is the +node's consolidated `LocalDb:Path` database (see +[Configuration.md](Configuration.md)). + +> **Where the queue lives (changed).** The buffer is a table — `alarm_sf_events` — in the node's +> consolidated LocalDb, not a standalone `alarm-historian.db`. That is what lets it **replicate to +> the redundant pair peer**, so undelivered alarm history survives losing the node holding it. +> Two consequences follow, both covered in the +> [pair-replication runbook](operations/2026-07-20-localdb-pair-replication.md): +> **only the Primary drains** (the Secondary holds a full replica and would otherwise re-send +> everything), and **delivery is at-least-once across a failover** by design. On first boot after +> the upgrade, `AlarmSfLegacyMigrator` copies any existing `alarm-historian.db` across and renames +> it `.migrated`. ```json { "AlarmHistorian": { "Enabled": true, - "DatabasePath": "C:\\ProgramData\\OtOpcUa\\alarmhistorian.db", "BatchSize": 100, "DrainIntervalSeconds": 5, "Capacity": 1000000, @@ -194,8 +251,7 @@ section (see [Historian.md](Historian.md)). | Key | Type | Default | Description | |---|---|---|---| -| `Enabled` | bool | `false` | Enable the SQLite store-and-forward sink (drains to the HistorianGateway `SendEvent` path). `false` → `NullAlarmHistorianSink`. | -| `DatabasePath` | string | `alarm-historian.db` | Path to the SQLite queue file. Created on first use (WAL mode). Set an **absolute** path in production. | +| `Enabled` | bool | `false` | Enable the durable store-and-forward sink (drains to the HistorianGateway `SendEvent` path). `false` → `NullAlarmHistorianSink`. Requires LocalDb to be registered — an enabled sink with no `ILocalDb` fails at resolution rather than silently discarding events. | | `BatchSize` | int | `100` | Max rows per drain cycle handed to `IAlarmHistorianWriter.WriteBatchAsync`. | | `DrainIntervalSeconds` | int | `5` | Seconds between drain-worker ticks. | | `Capacity` | long | `1000000` | Max queued rows before the sink evicts the oldest (data-loss signal via `EvictedCount`). | @@ -207,6 +263,15 @@ section (see [Historian.md](Historian.md)). > `RuntimeDb:EventReadsEnabled=true`. The old Wonderware connection keys (`SharedSecret` / > `AlarmHistorian:Host`/`Port`/`UseTls`/`ServerCertThumbprint`) were pruned. +> **`DatabasePath` was removed.** The queue no longer owns a file. The key is still *read*, by +> `AlarmSfLegacyMigrator`, purely to locate a pre-consolidation `alarm-historian.db` to migrate — +> so leave it in place through the upgrade, then delete it once every node has an +> `alarm-historian.db.migrated` sidecar. A leftover value configures nothing. + +> **`DrainState` gained `NotPrimary`.** A gated-off Secondary reports it instead of `Idle`, so a +> rising queue depth there is distinguishable from a stalled drain. Both nodes reporting it means +> nobody is draining — see the runbook. + > Dev and docker-dev deployments leave `Enabled` unset (defaults to `false`) so alarm transitions historize to nowhere unless a HistorianGateway is configured. --- @@ -221,3 +286,5 @@ section (see [Historian.md](Historian.md)). - [ScriptedAlarms.md](ScriptedAlarms.md) — the scripted-alarm engine that emits most events into this sink. - [ServiceHosting.md](ServiceHosting.md) — the external HistorianGateway backend. +- [operations/2026-07-20-localdb-pair-replication.md](operations/2026-07-20-localdb-pair-replication.md) + — where the queue lives, the Primary-only drain, and the one-time legacy migration. diff --git a/docs/AlarmTracking.md b/docs/AlarmTracking.md index 4f00551d..ce5e3c63 100644 --- a/docs/AlarmTracking.md +++ b/docs/AlarmTracking.md @@ -365,12 +365,16 @@ AB CIP ALMD) route to AVEVA Historian via the HistorianGateway: - `IAlarmHistorianSink` is the DI-registered intake contract. The default binding is `NullAlarmHistorianSink` (registered in `ServiceCollectionExtensions.AddOtOpcUaRuntime`). Production - deployments override it with `SqliteStoreAndForwardSink` wrapping + deployments override it with `LocalDbStoreAndForwardSink` wrapping `GatewayAlarmHistorianWriter` (the HistorianGateway `SendEvent` path) — see [ServiceHosting.md](ServiceHosting.md) for the HistorianGateway setup. -- `SqliteStoreAndForwardSink` queues each transition to a local - SQLite database and drains in the background via an - `IAlarmHistorianWriter`. **The durability guarantee is bounded**: the +- `LocalDbStoreAndForwardSink` queues each transition into the node's + consolidated LocalDb — where it **replicates to the redundant pair peer**, + so undelivered alarm history survives losing the node holding it — and + drains in the background via an `IAlarmHistorianWriter`. **Only the node + holding the Primary role drains**, since the peer holds a full replica; + delivery is consequently at-least-once across a failover, by design. See + [AlarmHistorian.md](AlarmHistorian.md). **The durability guarantee is bounded**: the queue capacity defaults to 1,000,000 rows; under a sustained historian outage, older non-dead-lettered rows are evicted (oldest first) to make room for new events. The `HistorianSinkStatus.EvictedCount` @@ -379,8 +383,9 @@ AB CIP ALMD) route to AVEVA Historian via the HistorianGateway: capacity, and dead-letter retention are tunable via the `AlarmHistorian` config section (`DrainIntervalSeconds`, `Capacity`, `DeadLetterRetentionDays`); `AlarmHistorianOptions.Validate()` logs a - startup warning for an empty `SharedSecret`, a relative `DatabasePath`, - or a non-positive knob. + startup warning for a non-positive knob. (It no longer warns about + `SharedSecret` or `DatabasePath` — both keys are gone: the connection + comes from `ServerHistorian`, and the queue lives in the LocalDb.) - `HistorianAdapterActor` (`src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs`) subscribes to the cluster `alerts` DPS topic, translates each diff --git a/docs/Configuration.md b/docs/Configuration.md index 5f01bfdc..c4b1ecb3 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -136,8 +136,9 @@ The historian backend is the external **`ZB.MOM.WW.HistorianGateway`** sidecar, `ZB.MOM.WW.HistorianGateway.Client` gRPC package (the retired Wonderware TCP sidecar is documented at [`docs/drivers/Historian.Wonderware.md`](drivers/Historian.Wonderware.md)). The OtOpcUa host reads three appsettings sections — `ServerHistorian` (read path + gateway connection), `ContinuousHistorization` -(FasterLog outbox + recorder draining to `WriteLiveValues`), and `AlarmHistorian` (SQLite store-and-forward -alarm sink draining to `SendEvent`). The gateway connection (endpoint / key / TLS) lives **only** in +(FasterLog outbox + recorder draining to `WriteLiveValues`), and `AlarmHistorian` (the store-and-forward +alarm sink draining to `SendEvent`; its buffer lives in the node's `LocalDb:Path` database, not a +file of its own). The gateway connection (endpoint / key / TLS) lives **only** in `ServerHistorian`; the other two sections source it from there. The gateway API key is supplied via the environment variable **`ServerHistorian__ApiKey`** — never committed diff --git a/docs/README.md b/docs/README.md index c5df5b74..6c320064 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,7 +30,7 @@ The project was originally called **LmxOpcUa** (a single-driver Galaxy/MXAccess | [Subscriptions.md](v1/Subscriptions.md) | Monitored items → `ISubscribable` + per-driver subscription refcount (v1 archive) | | [AlarmTracking.md](AlarmTracking.md) | `IAlarmSource` + `AlarmSurfaceInvoker` + OPC UA alarm conditions — native Galaxy alarms end-to-end (live) | | [AlarmTracking.md](v1/AlarmTracking.md) | Original alarm-tracking write-up (v1 archive) | -| [AlarmHistorian.md](AlarmHistorian.md) | `Core.AlarmHistorian` store-and-forward SQLite sink — `SqliteStoreAndForwardSink`, `IAlarmHistorianWriter`, dead-letter/retry/eviction | +| [AlarmHistorian.md](AlarmHistorian.md) | `Core.AlarmHistorian` store-and-forward sink — `LocalDbStoreAndForwardSink`, the replicated `alarm_sf_events` buffer + Primary-only drain, `IAlarmHistorianWriter`, dead-letter/retry/eviction | | [DataTypeMapping.md](v1/DataTypeMapping.md) | Per-driver `DriverAttributeInfo` → OPC UA variable types (v1 archive — live mapping is in `src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/DataTypeMap.cs`) | | [IncrementalSync.md](IncrementalSync.md) | Address-space rebuild on redeploy + `sp_ComputeGenerationDiff` | | [HistoricalDataAccess.md](v1/HistoricalDataAccess.md) | `IHistoryProvider` as a per-driver optional capability (v1 archive) | diff --git a/docs/Redundancy.md b/docs/Redundancy.md index 6692bed9..a0b5f3ad 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -69,6 +69,23 @@ Roles come from `RedundancyStateActor.BuildSnapshot`: a node with the `driver` role is `Primary` when it holds the `driver` role-leader lease, otherwise `Secondary`; a node without the `driver` role is `Detached`. +> **KNOWN LIMITATION — the election is per *Akka* cluster, not per application `Cluster`.** +> `RoleLeader("driver")` yields exactly **one** Primary across the whole Akka cluster. A fleet that +> runs several application clusters (each with its own redundant pair) inside one Akka cluster — +> which is what `docker-dev/docker-compose.yml` models, with MAIN, SITE-A and SITE-B — therefore has +> one Primary among *all* its driver nodes, not one per pair. Every consumer of the role inherits +> this: `ServiceLevel`, the `alerts` emit gate, and the inbound device-write gate below. +> +> The unit of redundancy everywhere else in the product is the application `Cluster` (`ClusterId`) — +> it owns drivers, devices and `ClusterNode` rows, and the LocalDb deployment-artifact cache is +> already keyed by it *so that a pair shares one entry*. Scoping the election the same way is the +> fix; `RedundancyStateActor` is an admin-role singleton in the same assembly as +> `AdminOperationsActor`, which already reads and groups `ClusterNodes` by cluster. +> +> Surfaced by the LocalDb Phase 2 live gate, where it stopped the alarm-history drain on every node +> in the fleet — see `docs/plans/2026-07-20-localdb-phase2-live-gate.md`. The alarm drain no longer +> depends on this (it defers only to a node that shares its queue), but the other three gates still do. + ## Data flow ``` @@ -221,15 +238,19 @@ Under warm/hot redundancy both cluster nodes run `ScriptedAlarmHostActor` and ev - **`alerts` topic emission** — `ScriptedAlarmHostActor` and `DriverHostActor.ForwardNativeAlarm` subscribe to the `redundancy-state` DPS topic and cache the local node's `RedundancyRole`, then gate the cluster `alerts` publish through `PrimaryGatePolicy` (table above). The OPC UA condition-node write and inbound ack/shelve command processing remain **ungated** on both nodes so the secondary is always ready to serve clients after a failover. - **`HistorianAdapterActor` historization** — likewise Primary-gated so alarm historization is exactly-once across all alarm sources. The actor subscribes to the `alerts` DPS topic and translates each `AlarmTransitionEvent` → `AlarmHistorianEvent` before enqueuing it on the sink; scripted alarms therefore historize exactly once regardless of cluster size. +- **The alarm sink's DRAIN** — a second, independent gate on the same decision. The enqueue gate above is not sufficient any more: since Phase 2 the store-and-forward buffer lives in the replicated LocalDb, so the Secondary holds a full copy of the Primary's queued rows and an ungated drain there would re-deliver every event continuously. `LocalDbStoreAndForwardSink` takes a `drainGate` fed from `IRedundancyRoleView` — a singleton `DriverHostActor` publishes its `PrimaryGatePolicy` verdict to, because the drain runs on a timer and cannot receive `RedundancyStateChanged` itself. A gated-off node reports `HistorianDrainState.NotPrimary`. Delivery is **at-least-once across a failover** by design: rows the old Primary delivered just before dying may not have replicated their deletes yet. See [AlarmHistorian.md](AlarmHistorian.md). Net effect: each alarm transition appears **once** on `/alerts` and would historize once, not once per node. See [ScriptedAlarms.md](ScriptedAlarms.md) and [AlarmTracking.md](AlarmTracking.md) for the scripted-alarm engine internals. -## Pair-local config cache (LocalDb — Phase 1) +## Pair-local store (LocalDb — Phases 1 + 2) Independently of the ServiceLevel machinery above, every **driver-role** node keeps a consolidated -[`ZB.MOM.WW.LocalDb`](../CLAUDE.md) SQLite database that caches the **deployed-configuration +[`ZB.MOM.WW.LocalDb`](../CLAUDE.md) SQLite database. Phase 2 added the **alarm store-and-forward +buffer** (`alarm_sf_events`) to it alongside the config cache, so a node that dies holding +undelivered alarm history no longer takes it to the grave — see the Primary-only drain note above +and [AlarmHistorian.md](AlarmHistorian.md). Phase 1 caches the **deployed-configuration artifact** (chunked). Its job is a single failure mode redundancy does not otherwise cover: a driver node restarting into a **central-SQL-Server outage** would come up with no configuration at all. With the cache, it **boots from the last artifact it applied** instead, and logs a running-from-cache diff --git a/docs/drivers/Historian.Wonderware.md b/docs/drivers/Historian.Wonderware.md index 839ca7fc..ebf18c0b 100644 --- a/docs/drivers/Historian.Wonderware.md +++ b/docs/drivers/Historian.Wonderware.md @@ -13,7 +13,7 @@ OtOpcUa now consumes the **`ZB.MOM.WW.HistorianGateway`** sidecar through the Gi - **HistoryRead** → `GatewayHistorianDataSource` over the `ServerHistorian` appsettings section. - **Alarm history** → `GatewayAlarmHistorianWriter` (the gateway `SendEvent` path) behind the durable - `SqliteStoreAndForwardSink`; alarm-history `ReadEvents` needs the gateway running + `LocalDbStoreAndForwardSink`; alarm-history `ReadEvents` needs the gateway running `RuntimeDb:EventReadsEnabled=true`. - **Continuous historization** → a crash-safe FasterLog outbox + `ContinuousHistorizationRecorder` draining to the gateway's `WriteLiveValues` (`ContinuousHistorization` section); needs the gateway diff --git a/docs/operations/2026-07-20-localdb-pair-replication.md b/docs/operations/2026-07-20-localdb-pair-replication.md index 78422482..8b7fc525 100644 --- a/docs/operations/2026-07-20-localdb-pair-replication.md +++ b/docs/operations/2026-07-20-localdb-pair-replication.md @@ -1,18 +1,20 @@ # LocalDb pair replication — operations runbook -> **Phase 1 scope.** Every driver-role OtOpcUa node keeps a consolidated -> [`ZB.MOM.WW.LocalDb`](../../CLAUDE.md) SQLite database. Today it caches exactly one thing: the -> **deployed-configuration artifact**, chunked, so a node can **boot from cache when central SQL -> Server is unreachable**. That cache can *optionally* replicate to the node's redundant pair peer -> over the library's gRPC sync. **Replication is default-OFF and fail-closed.** This runbook covers -> enabling it, the operational rules that keep a pair converging, and the DB-inspection safety -> rules. +> **Scope.** Every driver-role OtOpcUa node keeps a consolidated +> [`ZB.MOM.WW.LocalDb`](../../CLAUDE.md) SQLite database. It holds two things: the +> **deployed-configuration artifact** (chunked, Phase 1) so a node can **boot from cache when +> central SQL Server is unreachable**, and the **alarm store-and-forward buffer** (Phase 2) so a +> node that dies holding undelivered alarm history does not take it to the grave. Both can +> *optionally* replicate to the node's redundant pair peer over the library's gRPC sync. +> **Replication is default-OFF and fail-closed.** This runbook covers enabling it, the operational +> rules that keep a pair converging, and the DB-inspection safety rules. ## What replicates, and what it buys you -- **Replicated tables:** `deployment_artifacts` (the artifact, split into base64 chunks) and - `deployment_pointer` (one current-deployment pointer per cluster). Registered — in this order, - after the DDL — by `LocalDbSetup.OnReady`. +- **Replicated tables:** `deployment_artifacts` (the artifact, split into base64 chunks), + `deployment_pointer` (one current-deployment pointer per cluster), and `alarm_sf_events` (the + alarm store-and-forward buffer). Registered — in this order, after the DDL — by + `LocalDbSetup.OnReady`. - **The payoff:** in a redundant driver pair, *either* node can be the one that applied the latest deploy and wrote the cache. With replication on, the peer holds a byte-identical copy. So a node that restarts into a central-SQL outage can boot the current config **even if it never applied @@ -22,6 +24,47 @@ deployed. Retention (newest-2 deployments per cluster) prunes on one node and the deletes replicate as tombstones. +## Alarm store-and-forward — what replication changes + +Phase 2 moved the alarm buffer out of its own `alarm-historian.db` and into this database. Three +consequences an operator needs to know: + +- **Only the Primary drains.** The buffer replicates, so the Secondary holds a full copy of the + Primary's queue. Its drain worker is gated on the same Primary decision the inbound-write and + native-ack gates use, and reports `DrainState = NotPrimary`. **A rising queue depth on a + Secondary is correct.** A rising queue depth on *both* nodes is not — see below. +- **Delivery is at-least-once across a failover, by design.** Rows the old Primary delivered + moments before it died may not have had their deletes replicated yet, so the new Primary will + re-send them. This is accepted, not a defect: a duplicate alarm-history row is recoverable, a + missing one is not. There is deliberately no dedup layer. +- **The buffer is bounded.** `AlarmHistorian:Capacity` (1,000,000 by default) still evicts the + oldest undelivered rows when the historian has been unreachable long enough. Watch + `EvictedCount` — non-zero means accepted alarm events were dropped before reaching the historian. + +### If BOTH nodes report `NotPrimary` + +Nobody is draining, and the queue is filling toward the capacity ceiling on both. The gate +default-DENIES while the redundancy role is unknown *and* a driver peer exists, so this is what a +redundancy snapshot that never names either node looks like — the identity-mismatch shape +`DriverHostActor` also warns about once ("redundancy snapshot omitted this node"). Check node +identity (`Cluster:PublicHostname` / `Cluster:Port` skew) before suspecting the sink. + +### One-time migration from `alarm-historian.db` + +On first boot after the upgrade, `AlarmSfLegacyMigrator` copies any pre-existing +`alarm-historian.db` into `alarm_sf_events` and renames the file to `alarm-historian.db.migrated`. + +- **Both nodes of a pair migrate independently**, and their files overlap — row ids are derived + from the event payload, so the same event on both nodes converges to one row rather than + duplicating. The new Primary drains the union. +- **The rename happens only after the copy commits.** A failure fails host startup with the legacy + file untouched; the `.migrated` sidecar is what makes a later boot a no-op. +- **A file whose shape is unrecognised is left alone** — not renamed, not copied — so it stays + available for inspection. +- The `AlarmHistorian:DatabasePath` key is **removed**. It is still *read* by the migrator, to + find the legacy file; it no longer configures anything. Delete it from appsettings once the + `.migrated` sidecar exists on every node. + ## Enabling replication on a pair Replication has two knobs, both under `LocalDb`: @@ -127,10 +170,22 @@ FROM __localdb_row_version WHERE table_name = 'deployment_pointer' ORDER BY pk_j -- Replication backlog (should drain to 0 when a pair is caught up) SELECT COUNT(*) FROM __localdb_oplog; + +-- Undelivered alarm history, and how much of it has given up +SELECT dead_lettered, COUNT(*) FROM alarm_sf_events GROUP BY dead_lettered; + +-- Why the dead-lettered rows died +SELECT alarm_id, attempt_count, last_attempt_utc, last_error +FROM alarm_sf_events WHERE dead_lettered = 1 ORDER BY last_attempt_utc DESC LIMIT 20; + +-- Convergence check for the buffer: identical dumps mean the peer holds the ORIGIN-stamped rows +SELECT pk_json, hlc, node_id, is_tombstone +FROM __localdb_row_version WHERE table_name = 'alarm_sf_events' ORDER BY pk_json; ``` ## See also - [`docs/Redundancy.md`](../Redundancy.md) — the pair-local config cache section. - [`docs/Configuration.md`](../Configuration.md) — the `LocalDb` appsettings section. +- [`docs/AlarmHistorian.md`](../AlarmHistorian.md) — the alarm sink, its knobs, and the drain. - The two-node convergence harness: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/`. diff --git a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json index 7cbfdc9a..2eb2139a 100644 --- a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json +++ b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json @@ -1,15 +1,86 @@ { "planPath": "docs/plans/2026-07-20-localdb-adoption-phase2.md", "tasks": [ - { "id": 0, "subject": "Task 0: Recon — sink schema, seam, drain lifecycle, role-view bridge (STOP conditions)", "status": "pending" }, - { "id": 1, "subject": "Task 1: alarm_sf_events schema + registration (+ exact-set pin update)", "status": "pending", "blockedBy": [0] }, - { "id": 2, "subject": "Task 2: Rewire sink onto ILocalDb + delete bespoke file management (cutover 1/2)", "status": "pending", "blockedBy": [1] }, - { "id": 3, "subject": "Task 3: Primary-gated drain via PrimaryGatePolicy (cutover 2/2 — may co-commit with Task 2)", "status": "pending", "blockedBy": [2] }, - { "id": 4, "subject": "Task 4: One-time alarm-historian.db legacy migrator", "status": "pending", "blockedBy": [3] }, - { "id": 5, "subject": "Task 5: Convergence + failover scenarios in the pair harness (+ positive control)", "status": "pending", "blockedBy": [3] }, - { "id": 6, "subject": "Task 6: Rig config + docs", "status": "pending", "blockedBy": [3] }, - { "id": 7, "subject": "Task 7: DoD sweep (offline) — STOP and report after this", "status": "pending", "blockedBy": [4, 5, 6] }, - { "id": 8, "subject": "Task 8: Live gate on the docker-dev rig (needs explicit user go-ahead)", "status": "pending", "blockedBy": [7] } + { + "id": 0, + "subject": "Task 0: Recon \u2014 sink schema, seam, drain lifecycle, role-view bridge (STOP conditions)", + "status": "completed", + "note": "STOP condition does NOT fire (no BLOB; PayloadJson TEXT). Recon doc: docs/plans/2026-07-20-localdb-phase2-recon.md. Key finding: the drain worker is an internal Timer inside the sink (not a hosted service/actor), so the gate is a Func ctor param; and HistorianAdapterActor already primary-gates ENQUEUE with a different policy (ShouldHistorize) - which is why today's ungated drain is safe and why replication breaks it. Deviations D-1..D-4 recorded." + }, + { + "id": 1, + "subject": "Task 1: alarm_sf_events schema + registration (+ exact-set pin update)", + "status": "completed", + "blockedBy": [ + 0 + ], + "note": "alarm_sf_events created in AlarmSfSchema (Core.AlarmHistorian) + registered third in LocalDbSetup.OnReady. Exact-set pin updated to 3 tables. DEVIATION D-5: kept the legacy delete-on-ack + dead_lettered flag + last_error rather than the plan's status column (no sweeper for 'delivered' rows; last_error is the only record of why a row died). Drain ORDER BY moves to (enqueued_at_utc, id)." + }, + { + "id": 2, + "subject": "Task 2: Rewire sink onto ILocalDb + delete bespoke file management (cutover 1/2)", + "status": "completed", + "blockedBy": [ + 1 + ], + "note": "Sink rewritten as LocalDbStoreAndForwardSink over ILocalDb; bespoke file/pragma/schema management deleted with the old class. AlarmHistorian:DatabasePath removed (breaking config key). Ids are a deterministic payload hash (D-1), not GUIDs." + }, + { + "id": 3, + "subject": "Task 3: Primary-gated drain via PrimaryGatePolicy (cutover 2/2 \u2014 may co-commit with Task 2)", + "status": "completed", + "blockedBy": [ + 2 + ], + "note": "Drain gated on IRedundancyRoleView, a singleton DriverHostActor publishes PrimaryGatePolicy's verdict to on every snapshot. Fails closed on a throwing gate; seeded OPEN so a non-redundant deployment is never silently stopped. New HistorianDrainState.NotPrimary + transition-logged (D-2). Landed with Task 2 as one commit (D-3)." + }, + { + "id": 4, + "subject": "Task 4: One-time alarm-historian.db legacy migrator", + "status": "completed", + "blockedBy": [ + 3 + ], + "note": "AlarmSfLegacyMigrator runs LAST in OnReady (which now takes IConfiguration - no skip-migration overload exists). DEVIATION D-6: ids are the payload hash (AlarmSfSchema.DeriveId, lifted out of the sink) rather than mig-{node}-{legacyId} - a warm pair's two legacy files OVERLAP, and node-prefixing would carry that duplication forward forever. DEVIATION: tests live in Host.IntegrationTests; the plan's Host.Tests project does not exist." + }, + { + "id": 5, + "subject": "Task 5: Convergence + failover scenarios in the pair harness (+ positive control)", + "status": "completed", + "blockedBy": [ + 3 + ], + "note": "4 scenarios green. POSITIVE CONTROL RUN (RegisterReplicated(alarm_sf_events) commented out): 3/4 went red immediately; the 4th (same-event-on-both-nodes) passed VACUOUSLY - with replication off each node trivially held its own single row. Strengthened by enqueuing a second distinct event on B and asserting BOTH nodes hold 2; it then went red under the control too. Control restored, all 4 green. Also fixed a SECOND exact-set pin the plan predicted: LocalDbWiringTests." + }, + { + "id": 6, + "subject": "Task 6: Rig config + docs", + "status": "completed", + "blockedBy": [ + 3 + ], + "note": "Rig: AlarmHistorian__Enabled=true on site-a-1/2 AND site-b-1/2, with MaxAttempts raised to 1e6 (D-4: the default 10 would dead-letter the queue ~5min into the rig's permanent gateway outage) and a deliberately unresolvable ServerHistorian__Endpoint. NOTE: the endpoint is REQUIRED - ServerHistorianOptionsValidator is consumer-gated on AlarmHistorian:Enabled, so an empty one fails host start. Docs: runbook, AlarmHistorian.md, AlarmTracking.md, Configuration.md, Redundancy.md, docs/README.md, CLAUDE.md." + }, + { + "id": 7, + "subject": "Task 7: DoD sweep (offline) \u2014 STOP and report after this", + "status": "completed", + "blockedBy": [ + 4, + 5, + 6 + ], + "note": "Build: 0 errors solution-wide; 0 warnings from every project this branch touches (the ~816 solution-wide warnings are pre-existing xUnit1051/OTOPCUA0001/CS86xx in untouched driver + client test projects). GREPS FOUND REAL DRIFT Task 6 missed: 8 live sites still named the deleted SqliteStoreAndForwardSink - CLAUDE.md, the AdminUI /alarms/historian panel text (user-visible), HistorianAdapterActor (a that did NOT warn), Runtime + Host + 2 Driver.Historian.Gateway doc comments, 1 gateway test comment, docs/drivers/Historian.Wonderware.md; plus docs/AlarmTracking.md still claimed a Validate() startup warning for a relative DatabasePath (that branch is gone). All fixed. Code refs to AlarmHistorian:DatabasePath now reduce to exactly two intentional ones: AlarmSfLegacyMigrator.LegacyPathKey and its test. No 'new SqliteConnection' in Core.AlarmHistorian. Both exact-set pins assert the 3-table set, both directions. Guard-deletion evidence (both vacuous passes + the pin inventory) consolidated into the recon doc." + }, + { + "id": 8, + "subject": "Task 8: Live gate on the docker-dev rig (needs explicit user go-ahead)", + "status": "completed", + "blockedBy": [ + 7 + ], + "note": "Gate doc: docs/plans/2026-07-20-localdb-phase2-live-gate.md. Checks 1/2/5/6 PASS; checks 3+4 NOT SATISFIED (see below). FOUR production defects found + fixed, three of which crash-looped every driver node before check 1 could run: (1) empty ServerHistorian:ApiKey crashes the host - the validator had explicitly classified it as 'degrades', which is false because the gateway client validates its own options at construction; (2) UseTls vs endpoint scheme mismatch crashes likewise, both directions; (3) plaintext h2c was UNREACHABLE - the adapter forwarded TLS-only options unconditionally so every documented http:// deployment crashed; (4) THE BLOCKER - the drain gate deferred to a CLUSTER-WIDE elected Primary while the queue is PAIR-LOCAL. On the rig that Primary is central-1 (carries the driver Akka role, replicates nobody, runs no historian), so ALL FOUR driver nodes suspended their drains including unpaired site-b: silent permanent loss where pre-Phase-2 drained fine. Fixed in 3 layers (separate ShouldDrainAlarmHistory policy; peer-host matching in DriverHostActor; gate short-circuited entirely when replication is unconfigured, testing BOTH PeerAddress and SyncListenPort since only the dialer sets the former). Also fixed a THIRD vacuous test: AwaitAssert on the seeded-open value passed with the guard deleted; now asserts the sequence of PUBLISHED values via a recording view. Migration evidence: 11 legacy rows across two overlapping files -> exactly 9 identical rows on both nodes, proving D-6's payload-hash identity live. OPEN DESIGN FORK (in the gate doc): a pair cannot identify its own Primary, so both halves drain - safe (no loss, duplicates only) but the gate's de-dup benefit is unrealised." + } ], - "lastUpdated": "2026-07-20T00:00:00Z" + "lastUpdated": "2026-07-21T00:00:00Z" } diff --git a/docs/plans/2026-07-20-localdb-phase2-live-gate.md b/docs/plans/2026-07-20-localdb-phase2-live-gate.md new file mode 100644 index 00000000..8328cca2 --- /dev/null +++ b/docs/plans/2026-07-20-localdb-phase2-live-gate.md @@ -0,0 +1,180 @@ +# LocalDb Phase 2 — live gate (docker-dev rig) + +> Task 8 of `2026-07-20-localdb-adoption-phase2.md`. Evidence log. +> Rig: local `docker-dev/docker-compose.yml` — 2 central + site-a pair (replication ON) + site-b pair +> (default-OFF), one Akka cluster, one shared SQL. +> +> **Safety rules followed** (same as the Phase 1 gate): all DB inspection via `docker cp` of the +> `db`/`-wal`/`-shm` triplet out of the container, querying the copy — never host `sqlite3` on the +> live WAL file. + +## Headline + +**The gate did not get past its own first boot before finding real defects.** Four production bugs, +all fixed on-branch with regression tests, plus a fifth finding in the test suite itself. Three of +the four crash-looped every driver node; the fourth silently stopped alarm history everywhere. + +Checks 1, 2, 5 and 6 pass. **Checks 3 and 4 are NOT satisfied** — see "The blocker" below; they +cannot be run as written on a rig that has no per-pair redundancy roles, and the defect they were +meant to confirm turned out to be the opposite of what the plan assumed. + +## Seeding the migration + +The rig had no pre-consolidation `alarm-historian.db`, so one was synthesised per node and `docker +cp`-ed to `/app/alarm-historian.db` (the WORKDIR-relative default) into created-but-not-started +containers — the exact upgrade shape. + +Deliberately overlapping content, to test D-6 in production: **6 rows on site-a-1, 5 on site-a-2, +2 of them byte-identical payloads** (the warm-pair duplication `HistorianAdapterActor` produces in +every boot window). 11 legacy rows, **9 distinct**. + +## Defects found and fixed + +### 1. Empty `ServerHistorian:ApiKey` crash-loops the host (was classified "degrades") + +`AlarmHistorian:Enabled=true` requires the gateway alarm writer, which requires a key. +`HistorianGatewayClientOptions.Validate()` throws `The gateway API key must not be empty` inside the +client factory during Akka startup, so every driver node crash-looped. + +`ServerHistorianOptionsValidator` exists precisely to convert that class of failure into a named +`OptionsValidationException` — but its documented fail tier explicitly excluded `ApiKey`, reasoning +that a keyless client "degrades rather than crashes (the gateway rejects calls)". **That premise is +false**: the client validates its own options at construction, so the process dies before any call. +Fixed; the key is never echoed in the message. + +### 2. `UseTls` / endpoint-scheme mismatch crash-loops the host + +Immediately after fixing #1, the rig crash-looped again: `UseTls` defaults to `true`, the rig +endpoint is `http://`, and the client throws `UseTls requires an https gateway endpoint`. The +inverse (`https://` + `UseTls=false`) throws too — both strings confirmed in the shipped client +assembly. Setting an `http` endpoint without clearing `UseTls` is an ordinary migration slip that +takes the host down. Now validated in both directions. + +### 3. Plaintext h2c was *unreachable* — every `http://` deployment crashed + +Third crash-loop, and a genuine product bug rather than operator config. +`HistorianGatewayClientAdapter.Create` forwarded the TLS-only options unconditionally, and +`AllowUntrustedServerCertificate` defaults to `false`, so it always sent +`RequireCertificateValidation=true` — which the client rejects outright when `UseTls=false` +(`RequireCertificateValidation is a TLS-only option and requires UseTls=true`). + +CLAUDE.md and `docs/Historian.md` both document `http://` as the supported way to select h2c, but +**no h2c configuration could start**, and the only workaround was to claim +`AllowUntrustedServerCertificate=true` — a certificate posture for a connection with no certificate. +Fixed: TLS-only options stay at their defaults when `UseTls=false`. + +### 4. THE BLOCKER — the drain gate deferred to a Primary that cannot deliver + +With the rig finally up, **every driver node logged `Historian drain suspended`. Nothing drained +anywhere** — including the two site-b nodes, which have no redundancy peer and no replication at all. +Before Phase 2 this deployment drained fine, because the drain was ungated. + +Root cause, established from the rig rather than guessed: + +- `RedundancyStateActor` derives roles from Akka's **cluster-wide** `RoleLeader("driver")`. +- `central-1`/`central-2` carry **both** `admin` and `driver` Akka roles (`OTOPCUA_ROLES=admin,driver`). +- So the elected driver Primary is **central-1** — a node that replicates nobody's LocalDb and does + not even run the alarm historian. +- Every site node was `Secondary`, saw that a Primary existed, and stood down in its favour. + +**The redundancy role is a cluster-wide election; the alarm queue is pair-local.** Deferring on +"somebody is Primary" is therefore wrong in principle: in a multi-pair fleet the Primary is usually +in another pair and holds none of this node's rows. The consequence is not a duplicate — it is the +buffer growing on every node until the capacity wall evicts the oldest events, i.e. silent permanent +loss of the audit trail the queue exists to protect. + +Fixed in three layers, each with a positive control: + +1. `PrimaryGatePolicy.ShouldDrainAlarmHistory(role, queueSharingPeerIsPrimary)` — a **separate** + policy from `ShouldServiceAsPrimary`. Unknown role ⇒ drain. A node stands down only for a Primary + that holds its rows. The two gates now deliberately disagree, and a test pins that disagreement so + nobody "harmonises" them back. +2. `DriverHostActor` publishes that verdict, matching the snapshot's Primary against this node's + configured LocalDb replication peer host. +3. `AddAlarmHistorian` short-circuits the gate entirely when replication is not configured — a node + whose rows exist nowhere else must never defer. The predicate tests **both** `Replication:PeerAddress` + and `SyncListenPort`, because only the dialing half sets the former and both halves share the queue. + +The asymmetry of the error costs is what drives every one of these: a false allow costs a duplicate +history row, which the design already accepts (*at-least-once across a failover, by design*, and +payload-hash ids collapse identical events); a false deny loses data silently. + +### 5. A third vacuous test — `AwaitAssert` on the seeded value + +Deleting the guard (publishing the old write-gate verdict) left `DriverHostActorRoleViewTests` +**green**. `AwaitAssert` polls until an assertion passes, and the assertion was "the view reads +open" — which is also the value the view is *seeded* with, so it succeeded at the first poll, before +the actor had processed anything. It could not distinguish a correct publish from no publish at all. + +Fixed by asserting on the **sequence of published values** via a recording view, never on current +state. Re-run under the control: red, for exactly the cases that matter. This is the same +"absence-assertion passes by race" trap recorded for #485/#486, in a new disguise. + +## Checks + +| # | Check | Result | Evidence | +|---|---|---|---| +| 1 | Migration ran; `.migrated` sidecar on both site-a nodes; row counts match | ✅ PASS | `alarm-historian.db.migrated` present on both. 11 legacy rows across the two files → **exactly 9 rows on each node**, id sets byte-identical — the 2 shared payloads collapsed, proving D-6's payload-hash identity in production. Each node holds the *other's* rows (`legacy/a1-only-*` present on a-2 and vice versa). Legacy `attempt_count` distinction survived the copy (the 3-attempt row still leads the 0-attempt rows), `last_error` carried, 0 dead-lettered | +| 2 | Alarm burst converges: identical rowsets, oplog drains to 0, dead letters 0 | ✅ PASS | Identical id sets on both nodes; `__localdb_oplog` depth **0** on both; **0** dead-lettered. Drain visibly live against the deliberately unresolvable gateway — `attempt_count` climbing 33 → 42 → 168 with `last_error='retry-please'`, which is exactly the buffering behaviour the queue exists for | +| 3 | Only the primary drains | ❌ **NOT SATISFIED** — see defect 4 | As written the check assumes a per-pair Primary exists. On this rig the elected Primary is a central node outside both pairs, so the honest post-fix behaviour is that **all four nodes drain**. No node can currently identify a queue-sharing Primary: site-b is unpaired, and of the site-a pair only the dialing half knows its peer. Safe (no loss, duplicates only), but the check's intent is unproven | +| 4 | Failover: standby resumes draining; count double-delivered events | ⛔ **NOT RUN** | Superseded by defect 4. A meaningful failover test needs a rig where the pair itself elects a Primary; measuring "double delivery" is meaningless while both halves drain unconditionally | +| 5 | Both nodes restarted together: clean rejoin, identical counts, no I/O errors | ✅ PASS | `docker compose restart site-a-1 site-a-2` → **0** `disk I/O error` / `SQLITE_IOERR` / `corrupt` lines and 0 fatals in either log; both back to 9 rows / 0 dead / oplog 0, still identical | +| 6 | site-b default-OFF pin: sink works locally, no sync traffic | ✅ PASS | `alarm_sf_events` created and registered on both site-b nodes; port **9001 not bound** (`/proc/net/tcp`), no sync listener, no replication config; both drain their own queue (defect 4's third fix) with 0 fatals | + +## The open design question + +The gate leaves one question that is a design fork, not a bug to patch: + +**A pair cannot currently identify its own Primary.** Redundancy roles are elected cluster-wide, but +the queue is pair-local. Only the dialing half of a pair knows its peer (`Replication:PeerAddress`); +the listening half sets `SyncListenPort` only. So the drain gate can never fully engage, and both +halves of a replicated pair drain — safe, but duplicated. + +### Follow-up analysis — the root cause is wider than the drain gate + +Two things assumed while writing the section above turned out to be wrong, and correcting them +changes the recommendation. + +**The rig is not misconfigured.** `central-1`/`central-2` carry `admin,driver` deliberately — the +compose says so in as many words ("central is admin,driver"), because they are themselves a +redundant pair that also runs the MAIN cluster's drivers. Stripping the driver role would change what +the rig models, not fix anything. + +**A "pair" is not a missing concept — it already exists, as the application Cluster.** OtOpcUa's +`Cluster`/`ClusterId` (the config-DB entity that owns drivers, devices and `ClusterNode` rows) is +exactly the unit a redundant pair belongs to. Phase 1 already relies on this: the deployment-artifact +cache is *keyed by ClusterId so a pair shares one entry*, and the rig's three application clusters +(MAIN, SITE-A, SITE-B) are precisely its three pairs. + +So the real defect is that **`RedundancyStateActor` elects one Primary per *Akka* cluster, while +every meaningful unit of redundancy is an *application* cluster.** The rig runs six driver nodes +across three application clusters in one Akka cluster — a topology the product otherwise supports +throughout — and exactly one of those six can ever be Primary. + +That mis-scoping is not confined to the alarm drain. Every consumer of the role inherits it: +`ServiceLevel`, the `alerts` emit gate, and the inbound device-write gate. On a multi-cluster fleet +today, only one driver node cluster-wide will service Primary-gated work at all. + +### Revised recommendation + +**(C), as a separate change with its own live gate — and explicitly *not* (A).** + +- (C) is no longer "invent a pair concept"; it is "elect per `ClusterId` instead of per Akka cluster", + using a first-class entity that already exists. It is feasible where it needs to be: + `RedundancyStateActor` has no DB access today, but it is an admin-role singleton in the same + assembly as `AdminOperationsActor`, which already reads and groups `ClusterNodes` by cluster. +- **(A) is now actively undesirable.** Deriving pair identity from `LocalDb:Replication:PeerAddress` + would introduce a *second*, parallel notion of "who is my partner" that (C) immediately obsoletes — + and it cannot be done cleanly anyway: the library exposes no initiator/passive flag, so making both + halves declare a peer means both halves *dial*, opening two duplex sessions purely to obtain a + configuration value. +- (C) touches the inbound device-write gate, which is safety-critical (the archreview 03/S4 + dual-primary fix). It therefore belongs in its own change, with its own live gate, rather than + bolted onto Phase 2. + +Until then this branch is safe in every topology — no configuration loses data — so what remains is +reclaiming the de-duplication benefit, not an outstanding hazard. + +## Not merged + +Per the plan, nothing on this branch is merged. diff --git a/docs/plans/2026-07-20-localdb-phase2-recon.md b/docs/plans/2026-07-20-localdb-phase2-recon.md new file mode 100644 index 00000000..1d858f5a --- /dev/null +++ b/docs/plans/2026-07-20-localdb-phase2-recon.md @@ -0,0 +1,371 @@ +# LocalDb Phase 2 — Recon findings + +**Date:** 2026-07-21 · **Branch:** `feat/localdb-phase2` · **Base:** `master` `2e46d054` + +Answers Task 0 of `2026-07-20-localdb-adoption-phase2.md`. Every claim carries a `file:line` +citation against the base commit. Read the **Deviations** section last — two of them change the +shape of Tasks 2/3 from what the plan assumed. + +--- + +## 1. The sink's table schema + +**STOP condition does NOT fire** — confirmed by re-reading the executed DDL, as the plan's +pre-answer instructed. + +Executed DDL: `SqliteStoreAndForwardSink.cs:652-670` (`InitializeSchema`). It matches the class +doc-comment at `:17-26` exactly — no drift between documented and executed schema. + +```sql +CREATE TABLE IF NOT EXISTS Queue ( + RowId INTEGER PRIMARY KEY AUTOINCREMENT, + AlarmId TEXT NOT NULL, + EnqueuedUtc TEXT NOT NULL, + PayloadJson TEXT NOT NULL, + AttemptCount INTEGER NOT NULL DEFAULT 0, + LastAttemptUtc TEXT NULL, + LastError TEXT NULL, + DeadLettered INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS IX_Queue_Drain ON Queue (DeadLettered, RowId); +``` + +| Question | Answer | +|---|---| +| BLOB columns? | **None.** All 8 columns are TEXT/INTEGER. Payload is `PayloadJson TEXT NOT NULL`. No base64 conversion needed; the 171 KB chunk sizing is moot. | +| PK autoincrement? | **Yes** — `RowId INTEGER PRIMARY KEY AUTOINCREMENT`. The migrator must mint deterministic `mig-{node}-{legacyId}` ids and the new table needs a TEXT PK, as the plan specified. | +| Indices to carry | One: `IX_Queue_Drain (DeadLettered, RowId)` — the drain's covering index. `alarm_sf_events` wants the same shape over (status, insertion order). | +| Largest realistic payload | `PayloadJson` is a serialized `AlarmHistorianEvent` — 10 scalar fields (`AlarmHistorianEvent.cs`), no collections, no nesting. Low single-digit KB worst case; `Comment` is the only loosely-bounded field. | + +**Legacy column list** for the migrator's `pragma_table_info` intersection check: +`RowId, AlarmId, EnqueuedUtc, PayloadJson, AttemptCount, LastAttemptUtc, LastError, DeadLettered`. + +`RegisterReplicated` will accept this table: no BLOB, and the PK will be app-minted TEXT. + +## 2. The public seam + +`IAlarmHistorianSink` (`IAlarmHistorianSink.cs:23-34`) is **two members**: + +```csharp +Task EnqueueAsync(AlarmHistorianEvent evt, CancellationToken cancellationToken); +HistorianSinkStatus GetStatus(); +``` + +The concrete `SqliteStoreAndForwardSink` additionally exposes, and these are all consumed +somewhere, so the rewire must keep them: + +| Member | Cited | Consumer | +|---|---|---| +| `StartDrainLoop(TimeSpan)` | `:189` | `ServiceCollectionExtensions.cs:105` (DI factory) | +| `DrainOnceAsync(CancellationToken)` | `:320` | tests only — the deterministic drive | +| `RetryDeadLettered()` | `:511` | operator action (AdminUI) | +| `CurrentBackoff` | `:676` | tests | +| `DebugCapacityProbeCount` | `:99` | tests | +| `DefaultCapacity` / `DefaultMaxAttempts` consts | `:49`,`:53` | `AlarmHistorianOptions.cs:38,45` | +| `IDisposable` | `:679` | DI + tests; **disposes the injected writer too** | + +There is **no separate enqueue/dequeue/markDelivered/deadLetter interface** — the plan's item 2 +guessed at one. Dequeue, delivered-marking and dead-lettering are all *private* to the sink +(`ReadBatch :532`, `DeleteRow :564`, `DeadLetterRow :573`, `BumpAttempt :587`). That is good news: +the rewire is entirely internal to one class and the public seam is untouched. + +## 3. Semantics to preserve + +All defaults live in **two** places that must stay in agreement: + +| Semantic | Sink default | Options default | Enforced at | +|---|---|---|---| +| `Capacity` 1,000,000 | `:49` | `AlarmHistorianOptions.cs:38` | `EnforceCapacityFastPathAsync :271` → `EnforceCapacityAsync :602` | +| `MaxAttempts` 10 | `:53` | `:45` | drain loop `:429` | +| Dead-letter retention 30 d | `:50` | `:41` | `PurgeAgedDeadLetters :638`, run once per drain tick | +| `BatchSize` 100 | ctor arg `:116` | `:31` | `ReadBatch :542` | +| `DrainIntervalSeconds` 5 | — | `:34` | `StartDrainLoop` via `ServiceCollectionExtensions.cs:105` | + +Behaviours that are **not** in the plan's list but are load-bearing, and which the cutover must +not silently drop: + +- **Backoff ladder** 1s→2s→5s→15s→60s (`:57-64`), applied to the timer's *next due time* + (`RescheduleDrain :224`) so an outage genuinely slows the cadence. +- **Overflow evicts oldest non-dead-lettered rows** with a WARN and a lifetime + `EvictedCount` counter (`:630-635`) — the durability guarantee is explicitly bounded. +- **Corrupt-payload rows dead-letter immediately** for their own RowId so they cannot stall the + queue head (`:344-361`). +- **Cardinality guard**: a writer returning ≠1 outcome per event is treated as a batch retry + (`:393-404`). +- **In-memory row counter with periodic resync** every 10,000 enqueues (`:89-96`, `:271-290`) — + a perf optimisation that avoids `COUNT(*)` on the hot enqueue path. +- **Post-dispose throwing contract**: `GetStatus`/`RetryDeadLettered`/`EnqueueAsync`/`StartDrainLoop` + throw `ObjectDisposedException`; `DrainOnceAsync` returns quietly (`:322`). Pinned by a + regression test (`SqliteStoreAndForwardSinkTests.cs:854-865`). + +## 4. Drain-worker lifecycle — **the finding that reshapes Task 3** + +> The plan asked "hosted service or actor?". **Neither.** + +The drain worker is a **self-rescheduling one-shot `System.Threading.Timer` owned by the sink +itself** (`_drainTimer`, `:76`; started by `StartDrainLoop :189`; callback `DrainTimerCallback :199`; +re-armed by `RescheduleDrain :224`). It is started from inside the DI singleton factory at +`ServiceCollectionExtensions.cs:105`, so it begins ticking the moment the first consumer resolves +`IAlarmHistorianSink`. + +Consequences for the Primary gate: + +1. **The gate has to live inside `DrainOnceAsync`**, not in a wrapper. There is no external + scheduler to gate. +2. **`Core.AlarmHistorian` cannot see `PrimaryGatePolicy`.** The policy is in + `Runtime/Drivers/PrimaryGatePolicy.cs`, and Runtime → Core is the reference direction. + `Core.AlarmHistorian` references only `Core.Abstractions` (`.csproj`). So the sink takes a + **`Func` gate delegate** (default: always-drain) and Runtime supplies the lambda that + calls `PrimaryGatePolicy`. No new project reference, no new interface in Core. +3. **A role view is still needed** because the timer fires outside any actor. See the next point + for who should own it. + +### There is already a Primary gate — on the *enqueue* side, with a different policy + +`HistorianAdapterActor` (`Historian/HistorianAdapterActor.cs`) already subscribes to +`RedundancyStateChanged` (`:86`, handler `:145`), caches `_localRole` (`:42`), and gates the +enqueue on `ShouldHistorize()` (`:97`): + +```csharp +private bool ShouldHistorize() => _localRole is not (RedundancyRole.Secondary or RedundancyRole.Detached); +``` + +This is **not** `PrimaryGatePolicy` — it ignores driver-member count and defaults to *write* while +the role is unknown. Its comment (`:68-71`) explains why it exists: DPS fans the Primary's single +`alerts` publish to **every** node's adapter, so without the gate both nodes of a pair would +double-write. + +**This is why today's ungated drain is safe, and why Phase 2 breaks that.** Today the Secondary +never enqueues, so its queue is empty and an ungated drain has nothing to send. Once +`alarm_sf_events` replicates, the Primary's rows land in the Secondary's table — and an ungated +drain there would double-deliver *every* event, continuously, not just at failover. **The drain +gate is not a nicety; it is required by the same commit that registers the table.** + +### Who owns the role view + +`HistorianAdapterActor`, not `DriverHostActor`. Both are spawned under +`WithOtOpcUaRuntimeActors` (`ServiceCollectionExtensions.cs:353`) which `Program.cs:133` calls only +under `hasDriver` — as are `AddAlarmHistorian` (`:175`) and `AddOtOpcUaLocalDb` (`:185`). So sink, +LocalDb and adapter are co-located by construction. The adapter wins over `DriverHostActor` +because it **already** holds the role for exactly this purpose; using it adds no new subscription +and no new way for the view to go stale while the sink keeps draining. + +`DriverMemberCount` is read off Akka cluster state in `DriverHostActor.DriverMemberCount() :1446` +(try/catch → 0 on a non-cluster provider ⇒ single-node posture). Task 3 will lift that into a +small shared helper rather than copy it. + +## 5. Other `DatabasePath` construction sites + +- `ServiceCollectionExtensions.cs:98` — the only production site. +- `appsettings.json:57` — `"DatabasePath": "alarm-historian.db"`. +- `AlarmHistorianOptions.Validate() :54-55` — warns when the path is relative. **This warning + disappears with the key**; the validator loses one of its five checks. +- `SqliteStoreAndForwardSinkTests.cs` — ~30 construction sites, all passing the per-test temp + `_dbPath` seeded at `:25` and deleted in `Dispose :32`. These are the tests Task 2 rewires onto + a temp-file `ILocalDb`. +- **`docker-dev/docker-compose.yml` has no `AlarmHistorian` keys at all** — the rig runs with + `Enabled=false` today. Task 6 is therefore an *addition*, not an edit. + +No tooling or CLI constructs the sink. + +## 6. How `Enabled=false` short-circuits + +`AddAlarmHistorian` returns before registering anything when the section is absent or +`Enabled != true` (`ServiceCollectionExtensions.cs:87`), leaving the `NullAlarmHistorianSink` +seeded by `AddOtOpcUaRuntime :49`. The Null sink is a pure no-op (`IAlarmHistorianSink.cs:37-52`). + +Confirming the plan's own answer: **the tables are created unconditionally in `OnReady`** (cheap, +empty) because `AddOtOpcUaLocalDb` is independent of the `AlarmHistorian` gate. Only the +sink/drain stay Null. This is also required for correctness — `RegisterReplicated` must run on +both pair members regardless of their historian config, or a node that later enables the sink +would write rows that are never captured (see the CDC note below). + +## 7. Library constraints re-confirmed (README, `ZB.MOM.WW.LocalDb` 0.1.3) + +- **BLOB columns are rejected at registration.** Not a problem here (§1). +- **CDC semantics** — rows that existed *before* `RegisterReplicated` are neither captured nor + snapshotted. This is the mechanical reason the migrator must run **after** registration, and + the reason `LocalDbSetup.OnReady`'s ordering comment is load-bearing (`LocalDbSetup.cs:23-31`). +- **LWW keyed on the PK** — two nodes minting the same PK for *different* rows silently overwrite + each other; two nodes minting the same PK for the *same* row converge. Decision D-1 below turns + that second property into a feature. +- `ILocalDb` surface in use: `ExecuteAsync` / `QueryAsync(sql, rowMapper, params, ct)` / + `BeginTransactionAsync` / `CreateConnection` / `RegisterReplicated` + (`Deployment/LocalDbDeploymentArtifactCache.cs:77,82,205`). + +--- + +## Deviations and decisions + +Recorded here rather than raised as questions, per the plan's follow-up convention. + +### D-1 — Enqueue ids are a deterministic content hash, not `Guid.NewGuid()` + +**The plan says** GUIDs minted at enqueue (Task 2 step 2). + +**The problem.** The enqueue gate (`ShouldHistorize`, default-**write** on unknown role) and the +drain gate the plan specifies (`PrimaryGatePolicy`, default-**deny** on unknown role when paired) +disagree during the boot window. On a two-node pair before the first redundancy snapshot arrives, +**both** adapters enqueue the same DPS-fanned transition. With random GUIDs those are two distinct +rows that replicate into each other's tables — the buffer *duplicates* instead of converging, and +whichever node becomes Primary later delivers both copies. + +**The decision.** Mint `id` as a hash over the serialized payload. Two nodes independently +enqueuing the same fanned event produce the same PK, so LWW converges them to one row for free. +Two genuinely distinct events cannot collide: `AlarmHistorianEvent` carries a full-precision +`TimestampUtc` plus alarm id, kind, message and user, so an identical hash means an identical +event. + +This costs one `SHA256.HashData` per enqueue and removes a whole duplicate-row class. It also +slightly *improves* today's behaviour, where the same boot-window fan-out is already +double-delivered because the drain is ungated on both nodes. + +`ShouldHistorize` itself is left unchanged — out of scope, and with D-1 in place the two gates +disagreeing is no longer harmful. + +### D-2 — Gate denial must be observable + +`PrimaryGatePolicy` defaults to **deny** when the role is unknown on a multi-driver cluster. That +is the correct safety posture, but it means a redundancy-snapshot identity mismatch — the +documented 03/S5 shape that `DriverHostActor.cs:1485-1491` already warns about once — would +silently stop alarm history on *both* nodes, degrading only as retry/eviction growth minutes later. + +Task 3 will therefore: log the skip once per gate-state transition (not per tick — the drain runs +every 5 s), and surface it in `HistorianSinkStatus` so `/healthz` and the AdminUI can see +"buffering, not draining, because not Primary" rather than an unexplained rising queue depth. +Adding a `HistorianDrainState` member is additive; Task 3 will check every `switch` over it. + +### D-3 — Tasks 2 and 3 will land as one commit + +The plan anticipated this. Confirmed necessary: the gate delegate is a constructor parameter of +the rewritten sink, so the sink cannot compile in its final shape without it, and an intermediate +commit with a rewired-but-ungated sink would be a commit in which a replicated table is drained by +both nodes. Not a state worth having in history. + +### D-5 — The table keeps the legacy delete-on-ack shape, not a `status` column + +**The plan proposes** `status TEXT NOT NULL DEFAULT 'pending'` with values `pending | delivered | +dead`, and drops the legacy `LastError` column. + +**Two problems.** A `delivered` status means acknowledged rows accumulate in the table forever — +the plan specifies no sweeper for them, and they would also have to be excluded from the capacity +count, changing a semantic §3 says to preserve. And `LastError` is not dead weight: `DeadLetterRow` +writes the failure reason into it and `RetryDeadLettered` clears it, so it is the only operator- +facing record of *why* a row was dead-lettered. + +**The decision.** Keep the legacy shape: DELETE on acknowledgement, `dead_lettered` as the same 0/1 +flag, and retain `last_error`. Deleting keeps the table bounded with no second sweeper, and the +replication engine carries the delete as a tombstone (pruned after `TombstoneRetention`, 7 d), so +the peer drops its copy too — which is the property the plan wanted the `delivered` status for. +Columns then map 1:1 onto the legacy table, making the Task-4 migrator a straight copy. + +One genuine change: the drain's `ORDER BY` moves from `RowId ASC` to `enqueued_at_utc, id`, because +a hashed TEXT id carries no insertion order. Round-trip ("O") timestamps sort lexicographically in +chronological order, and `id` makes the ordering total; the index covers both. + +### D-6 — The migrator keys on the payload hash, not `mig-{node}-{legacyId}` + +**The plan says** deterministic ids of the form `mig-{NodeName}-{legacyId}`, mirroring ScadaBridge. + +**Why that is not enough here.** Node-prefixing does solve the collision the legacy AUTOINCREMENT +key would otherwise cause — node A's `RowId 7` and node B's `RowId 7` are different alarms — but it +*preserves* a duplication that ought to be collapsed. A warm pair's two legacy files **overlap**: +`HistorianAdapterActor` default-writes while its redundancy role is unknown, so both nodes accepted +the same fanned transitions during every boot window they ever had. Prefixed ids would carry every +one of those duplicates into the merged buffer permanently, for the eventual Primary to deliver +twice. + +**The decision.** Reuse D-1's `AlarmSfSchema.DeriveId` (lifted out of the sink so both call sites +share one identity rule). Distinct events cannot collide; identical ones converge. It also removes +the migrator's only need for node identity from configuration, and makes a crash between commit and +rename harmless under `INSERT OR IGNORE` — which the plan's own idempotency requirement wanted +anyway. + +### D-7 — The migrator's tests live in `Host.IntegrationTests` + +The plan names `tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests`. **That project does not exist.** The +tests sit beside `LocalDbSetupTests` in `Host.IntegrationTests`, which is where every other +LocalDb-facing test already lives. They are offline — a temp file and no external services. + +### D-4 — The live gate needs `MaxAttempts` raised on the rig + +Not actionable until Task 8, recorded now so it is not rediscovered there. The rig has no +HistorianGateway, so every drain attempt fails. With `MaxAttempts: 10` and the backoff ladder +capping at 60 s, buffered rows dead-letter roughly five minutes into the outage — which would make +the gate's "dead letters 0" check (step 2) fail for reasons that have nothing to do with +replication. Task 6 should set a high `MaxAttempts` on the rig, or Task 8 should assert on +dead-letter *timing* instead. + +Related: `Program.cs:170-173` shows an `AlarmHistorian:Enabled=true` / +`ServerHistorian:Enabled=false` deployment is explicitly supported and warned about — which is +exactly the rig's shape. Task 6 must confirm `ServerHistorianOptionsValidator` does not fail host +start for a disabled section with an empty `Endpoint`. + +--- + +## Guard-deletion evidence (recorded at the Task 7 DoD sweep) + +Every guard this phase adds was proved by deleting it and watching the tests go red. Two of those +runs found tests that would have passed over a broken implementation. Both are recorded here because +the *shape* of the vacuity is reusable, not just the fix. + +### Control 1 — remove `IRedundancyRoleView.Publish` from `DriverHostActor` + +**3 of 6** `DriverHostActorRoleViewTests` went red. The other three assert +`ShouldServiceAsPrimary == true`, which is exactly the value the view is **seeded** with — so they +cannot distinguish "published true" from "never published at all". That split is expected and +acceptable *given* the three that do go red pin the false cases, which only a real publish can +produce. Worth knowing before someone reads a future 3/6 result as a regression. + +### Control 2 — remove `RegisterReplicated("alarm_sf_events")` from `LocalDbSetup.OnReady` + +**3 of 4** `AlarmSfConvergenceTests` went red immediately. The fourth, +`TheSameEventOnBothNodes_ConvergesToOneRow`, **passed vacuously**: with replication switched off +each node trivially held its own single row, and "count == 1 on both nodes" was satisfied by two +completely unconverged databases. The assertion could not tell convergence from isolation. + +**Fix:** enqueue a second, *distinct* event on node B and require **both** nodes to hold two rows. +Isolation now yields 1 and 2; only convergence yields 2 and 2. Re-run under the control: red. +Control restored: all 4 green. + +The general trap — an assertion whose expected value is also what a *disabled* system produces — +is the same one recorded in the `#485`/`#486` unreadable-artifact defect class. + +### Exact-set replicated-table pins + +Two, not one, and both assert set equality (ordered `ShouldBe` against the literal three-table list, +so an added *or* dropped registration fails): + +- `LocalDbSetupTests` — drives the production `OnReady` callback directly. +- `LocalDbWiringTests` — resolves `ILocalDb` from the full driver DI graph, which is what catches a + registration that never reaches a running host. + +## Test-suite evidence (Task 7 DoD sweep) + +Full solution run on the branch, compared against a **full solution run on a detached worktree at +the pre-branch baseline `2e46d054`**. Comparing failure *sets*, not counts, because the suite has a +standing set of environment- and load-dependent failures that a raw count would hide. + +**The two failure sets are identical — all 13 tests, same names, both runs. Zero new failures.** + +| Assembly | Baseline | Branch | Delta | +|---|---|---|---| +| `Core.AlarmHistorian.Tests` | 0 failed / 29 passed | 0 failed / **32** passed | **+3** (the drain-gate tests; 29 ported from the old sink's suite) | +| `Runtime.Tests` | 1 failed / 418 passed | 1 failed / **424** passed | **+6** (`DriverHostActorRoleViewTests`) | +| `Host.IntegrationTests` | 2 failed / 169 passed | 2 failed / **186** passed | **+17** (migrator + convergence + pin updates) | + +The 13 standing failures, and why each is not this branch's: + +- **3 × `DriverTypeNamesGuardTests`** — `ArgumentNullException (secretResolver)` out of + `GalaxyDriverFactoryExtensions.Register`, reached by reflection. A real pre-existing defect in a + guard test, reproduced exactly on the baseline worktree. Unrelated to this phase. +- **4 × `AbLegacy.IntegrationTests`**, **3 × `OpcUaClient.IntegrationTests`**, + **1 × `DriverProbeHandshakeE2eTests.AbCip_Green_AgainstSim`** — driver fixtures on the + `10.100.0.35` docker host are not running. +- **1 × `RoslynVirtualTagEvaluatorTests.Evaluate_racing_ClearCompiledScripts_never_fails_with_disposed`** + — times out under full-suite load; passes in isolation. +- **1 × `ContinuousHistorizationRecorderTests.Retry_after_writer_failure_eventually_acks`** — fails + under full-suite load ("writer must have been called at least twice"), and `Runtime.Tests` passes + **425/425** when run alone. Also fails on the baseline. Worth noting for anyone reading a future + run: this one is *newly observed* here only because the previous sweep's log was truncated, not + because the branch introduced it. diff --git a/docs/plans/2026-07-21-per-cluster-mesh-design.md b/docs/plans/2026-07-21-per-cluster-mesh-design.md new file mode 100644 index 00000000..addeb661 --- /dev/null +++ b/docs/plans/2026-07-21-per-cluster-mesh-design.md @@ -0,0 +1,276 @@ +# Per-Cluster Akka Mesh — design + +> **Status:** design for review. No implementation planned yet. +> **Decisions settled 2026-07-21** — see §6: one central DB with driver nodes disconnected from it, +> the two-node keep-oldest gap accepted as ScadaBridge accepts it, and their auth posture matched for +> now. + +## 1. What this changes + +Today OtOpcUa runs **one Akka mesh containing every node of every application Cluster**. That was a +deliberate choice — `docs/plans/2026-06-07-per-cluster-scoping.md`, "Per-ClusterId Scoping +(hub-and-spoke **single mesh**)" — so the deploy channel could stay in-mesh: AdminUI → +`admin-operations` singleton → `deployments` topic → every node filters to its own `ClusterId`. + +The proposal is to move to the sister project's shape: **one Akka mesh per application Cluster, +two nodes maximum**, with central and clusters joined by explicit transports instead of cluster +gossip. + +The motivation is that redundancy becomes correct *by construction*. Every Primary-gated decision +(inbound device writes, native alarm acks, fleet alerts, ServiceLevel, and — since Phase 2 — the +alarm-history drain) currently asks "am I the driver Primary?" of a **cluster-wide** election, while +every resource being gated is **pair-local**. Splitting the mesh collapses the two scopes into one +and the question stops being ambiguous. + +## 2. What ScadaBridge actually does + +Researched directly from `/Users/dohertj2/Desktop/ScadaBridge`. Five things differ from the +summary in its own CLAUDE.md, and they matter: + +**Three transports cross the boundary, not two.** + +| Transport | Direction | Carries | +|---|---|---| +| Akka **ClusterClient** | central → site (+ replies) | command/control: deploy notify, lifecycle, queries, subscribe handshakes | +| **gRPC** server-streaming | **central dials into the site** | real-time attribute + alarm events | +| Plain **HTTP**, token-gated | site → central | the deployment config itself (notify-and-fetch) | + +**The gRPC direction is inverted from the obvious guess.** Data flows site→central, but *each site +node hosts the gRPC server* (Kestrel h2c :8083) and *central is the client*. There is no gRPC server +on central at all. + +**Active node = OLDEST Up member, explicitly not the cluster leader.** +`ActiveNodeEvaluator.SelfIsOldestUp` is documented as *"THE single definition of 'active node'"*: + +> *"Cluster LEADERSHIP (lowest address) is an Akka-internal concept that diverges from singleton +> placement permanently once the original first node restarts and rejoins; every product-level +> active/standby decision must use this evaluator, never `cluster.State.Leader`."* + +The equivalence **oldest-Up == where `ClusterSingletonManager` places singletons** *is* the design. + +**All clusters share one ActorSystem name** (`"scadabridge"`, hardcoded). They are separate clusters +only by seed-node partitioning — required, because Akka.Remote address matching means ClusterClient +could not reach a differently-named system. + +**Site nodes carry two roles:** `"Site"` and `"site-{SiteId}"`. Singletons scope to the +**site-specific** role. + +Other load-bearing details: + +- **Central discovers sites from the database** (a `Site` entity with `NodeAAddress`/`NodeBAddress` + + `GrpcNodeAAddress`/`GrpcNodeBAddress`), refreshed every 60 s and on admin change. Sites discover + central from **appsettings** — static, restart required. The asymmetry is deliberate. +- **Exactly one actor is exposed per side** via `ClusterClientReceptionist.RegisterService` — + `/user/central-communication` and `/user/site-communication` — registered **per node, not as a + singleton**, so contact rotation reaches whichever node answers. +- **No central buffering when a site is unreachable.** The send is dropped with a warning and the + caller's Ask times out. *"It keeps the central coordinator stateless with respect to site + availability."* A `ConnectionStateChanged` mechanism was built for this and **deleted as dead + code**. +- **Every unhandled message replies with a typed failure rather than dropping**, so a central Ask + can never stall on a missing handler. +- **Sender preservation is the whole Ask idiom**: `_client.Tell(new ClusterClient.Send(path, msg), + Sender)` routes the reply straight back to the waiting Ask, bypassing the comm actor. + +### The registered outage gap — read before copying + +> `Component-ClusterInfrastructure.md:113`: *"Only the FIRST seed listed in `Cluster:SeedNodes` may +> self-join to form a *new* cluster… a lone restarted non-first-seed node (with the first seed still +> down) loops on `InitJoin` forever — never `Up`, never routable. This is why the two-node +> keep-oldest **oldest/active-node crash is a total-outage gap**: after the oldest dies the younger +> survivor self-downs, and it cannot re-bootstrap alone. Recovery is operator-driven."* + +Their failover drill has a mode that exists *"to make the registered gap observable — not to pretend +it is covered."* **A two-node keep-oldest mesh has an acknowledged total-outage hole.** If OtOpcUa +adopts 2-node meshes, this must be decided deliberately, not inherited. + +### Both inter-cluster transports are unauthenticated + +Akka remoting has no TLS, no secure cookie, no `trusted-selection-paths`. The gRPC listener is h2c +with no auth covering `SiteStreamService` — including two Pull RPCs that return audit rows. The only +authenticated pieces are the HTTP fetch token and `LocalDbSyncAuthInterceptor`, which gates *only* +`/localdb_sync.v1.LocalDbSync/`. Their boundary assumes a trusted network. + +**OtOpcUa already ships that same fail-closed interceptor** (LocalDb Phase 1) — it is a ready-made +template if we choose to authenticate more than they did. + +## 3. What OtOpcUa has today + +**Nine cross-node DPS topics plus a singleton**, all currently relying on one mesh: + +| Channel | Direction | Purpose | +|---|---|---| +| `deployments` / `deployment-acks` | central ↔ nodes | deploy dispatch + acks | +| `driver-control` | central → nodes | AdminUI Reconnect/Restart | +| `redundancy-state` | central → nodes | roles + peer probe results | +| `alerts` | nodes → central | fleet alarms → `/alerts` | +| `driver-health`, `driver-resilience-status`, `fleet-status`, `script-logs` | nodes → central | AdminUI live panels | +| `admin-operations` singleton | central | operator ack/shelve into engines | + +Three facts make the split cheaper than that table suggests: + +1. **The deploy path is already notify-and-fetch.** `DispatchDeployment` carries only + `DeploymentId` + `RevisionHash` + `CorrelationId`; the node fetches the artifact itself. We + independently arrived at the pattern ScadaBridge had to retrofit, and we do **not** carry its + 128 KB Akka frame exposure on this path. +2. **Deploy state already has a DB substrate.** `ConfigPublishCoordinator` persists per-node ACKs to + `NodeDeploymentState` so a singleton failover recovers in-flight state from the DB. +3. **`ClusterNode` rows already enumerate every node per application Cluster** — the same table + `AdminOperationsActor` groups by cluster. This replaces the coordinator's one genuinely + mesh-bound dependency: it derives its expected-ack set from `Akka.Cluster.State.Members` filtered + by role. + +So the deploy channel needs only a small notify + ack transport. **The nine live-telemetry channels +are the real work**, and they are exactly what ScadaBridge built ClusterClient + gRPC for. + +## 4. A defect to fix regardless of this design + +OtOpcUa currently uses **two different node-selection rules at once**: + +- Split-brain resolution is **keep-oldest** (`Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf:49`). +- `ClusterSingletonManager` places singletons on the **oldest** member. +- But `RedundancyStateActor.BuildSnapshot` elects Primary from **`RoleLeader("driver")`** — lowest + address among driver members. + +Those select the same node only by coincidence. After a restart-and-rejoin the role leader and the +oldest member diverge permanently, which means the node the SBR protects and hosts every singleton +on need not be the node the data-plane gates consider Primary. This is precisely the rule ScadaBridge +forbids by name, and for the reason it gives: *"both sides claim leadership during a partition"* — +the dual-primary shape archreview 03/S4 exists to prevent. + +**Switching the role derivation to oldest-Up-with-role is small, is correct under either topology, +and should not wait for this design.** It also happens to be exactly what the separate-mesh model +needs, so it is not throwaway work. + +## 5. Target architecture + +Mirror ScadaBridge's shape, adapted to OtOpcUa's existing substrate: + +- **One Akka mesh per application `Cluster`, two nodes max.** Same ActorSystem name (`otopcua`) + everywhere; separation by seed-node partitioning only. +- **Roles per node:** `driver` plus a cluster-specific `cluster-{ClusterId}`, with singletons scoped + to the cluster-specific role. +- **Active node = oldest Up member with role** (§4), one definition, used by the data-plane gates, + ServiceLevel, and the alarm drain alike. +- **Central ↔ cluster command/control over ClusterClient**, with exactly one receptionist-registered + actor per side and central discovering cluster node addresses from `ClusterNode` rows (extended + with Akka + gRPC addresses, mirroring their `Site` entity). +- **Deploy stays notify-and-fetch, but the fetch changes source.** The notify crosses via + ClusterClient; the artifact then comes **from central**, not from the ConfigDb the driver node no + longer connects to (§6.1), and is cached in LocalDb. The coordinator's expected-ack set comes from + `ClusterNode` rows instead of cluster membership. +- **Driver nodes hold no ConfigDb connection.** LocalDb becomes their steady-state configuration + store; five current DB consumers are re-homed or re-sourced per §6.1. +- **Live telemetry over gRPC**, central dialling into each cluster node, replacing the seven + observability topics with one stream contract carrying a `oneof` event — additive-only field + evolution, contract-locked by test. + +**Copied:** their node-local configuration store fed by fetch-and-cache (§6.1), their oldest-Up +active-node rule (§4), their two-node keep-oldest posture including its acknowledged outage gap +(§6.2), and their unauthenticated inter-cluster transports (§6.3). + +**Deliberately not copied:** their *transient-only* central alarm cache. OtOpcUa persists alarm +history centrally through the historian, and Phase 2's replicated store-and-forward buffer already +guarantees delivery across a failover — so there is no reason to adopt a design whose stated +trade-off is that a new active node re-seeds alarm state from scratch. + +## 6. Decisions (settled 2026-07-21) + +### 6.1 DECIDED — one central DB, and driver nodes never connect to it + +**Verified in ScadaBridge before adopting.** `Program.cs` calls +`AddConfigurationDatabase(configDbConnectionString)` at line 264 — inside the Central branch +(89–478). The Site branch begins at line 479. **Site nodes never register the central configuration +database at all**; they receive config from central (notify → token-gated HTTP fetch) and persist it +locally. There is still exactly one authoritative configuration database — it is simply +central-only. + +OtOpcUa adopts the same shape: **the ConfigDb stays the single source of truth, and driver nodes stop +connecting to it.** They obtain their configuration from the central nodes and cache it locally in +LocalDb. + +**This promotes LocalDb from an outage cache to the driver node's steady-state configuration store.** +That is a reframing of the Phase 1 work rather than a contradiction of it: boot-from-cache stops +being the exceptional path and becomes the normal one, and "central SQL unreachable" stops being a +degraded mode for driver nodes because they never held a connection to lose. The chunking, +SHA-256 verification, newest-2 retention and pair replication all carry over unchanged. + +**Scope — five driver-side ConfigDb consumers must be re-homed or re-sourced.** Audited: + +| Consumer | Current use | Disposition | +|---|---|---| +| `DriverHostActor` | artifact fetch + ack writes (5 `CreateDbContext` sites) | artifact via fetch-and-cache; acks via the ack transport | +| `EfAlarmConditionStateStore` | Part 9 alarm condition state | **move to LocalDb** — it is pair-local state, the same journey Phase 2 made for the alarm S&F buffer | +| `DbHealthProbeActor` | DB reachability feeding ServiceLevel tiering | **semantics change** — a driver node has no DB to probe; decide what health input replaces it | +| `OpcUaPublishActor` | (audit required) | TBD | +| `ServiceCollectionExtensions` | registration | follows the above | + +`DbHealthProbeActor` deserves particular care: DB health is currently a ServiceLevel input, and +under this model it stops existing for driver nodes. That is a behaviour change on a client-visible +value, not just an internal refactor. + +### 6.2 DECIDED — accept the two-node keep-oldest gap, as ScadaBridge does + +**Status corrected.** What was resolved is the *documentation and the drill*, not the gap. Gitea +**PR #12** (closed) — — "R2-01: Cluster, +Host & Failover round-2 fixes" carried: + +- **T1** (`badc97af`) — the failover drill now kills the STANDBY by default, and its `active` mode + *"measures the registered keep-oldest outage instead of pretending recovery"*. +- **T2** (`d5364506`) — per-direction failover docs, dropping what the commit message calls *"the + undeliverable ~25s active-crash promise"*. + +So the closed work made the gap **honest**, not absent. `Component-ClusterInfrastructure.md:113` +still reads: *"Removing the gap itself is the **registered deferred keep-oldest topology/strategy +decision** (master tracker 2026-07-08, owner: user)."* There is no open Gitea issue for it — it is +tracked on that master tracker, not in the issue list. + +OtOpcUa inherits the same gap and, for now, the same posture. Worth recording so it is a known +accepted risk rather than a surprise: after the oldest node of a two-node mesh dies, the survivor +self-downs and cannot re-bootstrap alone, because only the first-listed seed may self-join. Recovery +is operator-driven — restart the dead first seed, or restart the survivor with a self-first seed +override. + +### 6.3 DECIDED — match ScadaBridge's auth posture for now + +Inter-cluster transports stay unauthenticated and unencrypted, as theirs are: the boundary assumes a +trusted network. Recorded as an accepted risk rather than an oversight, with two notes for whenever +it is revisited: `LocalDbSyncAuthInterceptor` is already in our tree as a fail-closed template, and +the surface most worth gating first is any Pull-style RPC that returns historical rows. + +## 7. Sequencing sketch + +Deliberately not a task plan — per-phase plans follow, one at a time. + +| Phase | Content | Independent of the split? | +|---|---|---| +| 0 | Oldest-Up role derivation (§4) | **Yes** — ship first | +| 1 | `ClusterNode` gains Akka + gRPC address columns; coordinator sources its expected-ack set from the DB | Yes | +| 2 | Comm actors + receptionist registration; ClusterClient transport; deploy notify + acks across the boundary | No | +| 3 | Config fetch-and-cache: artifact served by central, driver nodes read LocalDb (§6.1) | No | +| 4 | Cut the driver-side ConfigDb connection: re-home `EfAlarmConditionStateStore`, resolve the `DbHealthProbeActor` ServiceLevel input, audit `OpcUaPublishActor` | No | +| 5 | gRPC stream contract; migrate the seven observability topics | No | +| 6 | Mesh partition: per-cluster seed nodes, cluster-scoped roles + singletons; rig models the real topology | No | +| 7 | Failover drill (both directions, per §6.2); live gate | No | + +Phases 3 and 4 are the ones that change a running system's data path rather than its wiring, and each +deserves its own live gate. + +## 8. Risks + +- **LocalDb becomes load-bearing for configuration, not just resilient.** Phase 1 built it as a + fallback; §6.1 makes it the driver node's only config store. A cache defect that was previously a + degraded-mode bug becomes a total-availability bug — the #485 unreadable-artifact class, with the + blast radius raised. +- **`DbHealthProbeActor` feeds a client-visible value.** Removing the driver-side DB connection + changes what ServiceLevel means on those nodes; that is an interop-visible change, not internal. +- **The rig currently models the topology that would be abolished.** Phase 6 rewrites + `docker-dev/docker-compose.yml` substantially, and every live gate that depends on it. +- **Losing gossip loses free fleet-wide observability.** Seven AdminUI panels are fed by DPS today; + each needs an explicit stream and a reconnect story. +- **Akka frame size.** We are clean on the deploy path, but anything new that carries payload over + ClusterClient inherits the 128 KB default with `log-frame-size-exceeding` off — a silent + single-message drop that leaves the association healthy. If we send payload at all, set both. +- **Do not stack application-level last-write-wins on LocalDb's HLC.** Their `SiteStorageService` + carries this warning in code. Verified: our `deployment_pointer` upsert is unconditional and the + alarm sink is idempotent on a payload hash, so we are currently clean — keep it that way. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs new file mode 100644 index 00000000..7705dc02 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs @@ -0,0 +1,105 @@ +using System.Security.Cryptography; +using System.Text; +using Microsoft.Data.Sqlite; + +namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; + +/// +/// DDL for the alarm store-and-forward buffer: one row per alarm event awaiting delivery to +/// the historian gateway. +/// +/// +/// +/// Replaces the standalone alarm-historian.db the sink used to own outright. Living +/// in the consolidated LocalDb file is what lets the buffer replicate to the redundant pair +/// peer, so a node that dies with undelivered alarm history no longer takes it to the grave. +/// +/// +/// Deliberately depends on nothing but so it can be applied +/// to any connection — the host's LocalDbSetup.OnReady in production, and a bare +/// connection in a test — without dragging the DI graph along. Mirrors +/// DeploymentCacheSchema. +/// +/// +/// Why is TEXT and app-minted. The legacy queue keyed on +/// RowId INTEGER PRIMARY KEY AUTOINCREMENT. Convergence is last-writer-wins over the +/// primary key, so two nodes independently allocating rowid 7 for different alarms would +/// silently overwrite one another. The sink mints the id from a hash of the payload, which +/// additionally makes the same event converge to one row when both nodes of a pair enqueue +/// it — as they legitimately do in the window before the first redundancy snapshot arrives. +/// +/// +public static class AlarmSfSchema +{ + /// Table holding queued alarm events awaiting delivery. + public const string EventsTable = "alarm_sf_events"; + + /// The single primary-key column. + public const string IdColumn = "id"; + + /// + /// Derives a row's primary key from its serialized payload. + /// + /// + /// + /// Deterministic rather than a fresh GUID, so that the same event arriving twice + /// converges to one row under last-writer-wins instead of duplicating. That happens for + /// real in two places: HistorianAdapterActor default-writes while its redundancy + /// role is unknown, so both nodes of a pair accept the same fanned transition in the + /// window before the first snapshot; and both nodes independently migrate their own + /// legacy queue file, which for a warm pair holds overlapping history. + /// + /// + /// Two genuinely distinct events cannot collide. AlarmHistorianEvent carries a + /// full-precision timestamp alongside the alarm id, transition kind, message and user, + /// so an equal hash means an equal event. + /// + /// + /// The serialized AlarmHistorianEvent. + /// The row's primary key. + public static string DeriveId(string payloadJson) + { + ArgumentNullException.ThrowIfNull(payloadJson); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(payloadJson))); + } + + /// + /// Creates the buffer table if it does not already exist. Idempotent. + /// + /// + /// An already-open connection. ILocalDb.CreateConnection() hands out open, + /// pragma-configured connections — do not call Open() on one. + /// + public static void Apply(SqliteConnection connection) + { + ArgumentNullException.ThrowIfNull(connection); + + using var cmd = connection.CreateCommand(); + + // Columns map 1:1 onto the legacy Queue table so the one-time migrator is a straight copy + // and the drain keeps its existing semantics: dead_lettered is the same 0/1 flag, and an + // acknowledged row is DELETEd rather than marked. Deleting keeps the table bounded without + // a second sweeper, and the replication engine carries the delete as a tombstone, so the + // peer drops its copy too. + // + // The drain index orders by enqueued_at_utc because a hashed TEXT id carries no insertion + // order the way the legacy AUTOINCREMENT RowId did. Timestamps are round-trip ("O") format, + // so lexicographic ordering is chronological; id breaks ties so the order is total and the + // index covers the drain's read. + cmd.CommandText = """ + CREATE TABLE IF NOT EXISTS alarm_sf_events ( + id TEXT NOT NULL PRIMARY KEY, + alarm_id TEXT NOT NULL, + enqueued_at_utc TEXT NOT NULL, + payload_json TEXT NOT NULL, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_attempt_utc TEXT NULL, + last_error TEXT NULL, + dead_lettered INTEGER NOT NULL DEFAULT 0 + ); + CREATE INDEX IF NOT EXISTS ix_alarm_sf_events_drain + ON alarm_sf_events (dead_lettered, enqueued_at_utc, id); + """; + cmd.ExecuteNonQuery(); + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/IAlarmHistorianSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/IAlarmHistorianSink.cs index 7efaf3c9..72287399 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/IAlarmHistorianSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/IAlarmHistorianSink.cs @@ -4,13 +4,13 @@ namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; /// The historian sink contract — where qualifying alarm events land. Ingestion routes /// through the HistorianGateway alarm writer (the gateway's SendEvent gRPC path) /// behind the durable store-and-forward queue. Tests use an in-memory fake; production uses -/// . +/// . /// /// /// /// is fire-and-forget from the engine's perspective — /// the sink MUST NOT block the emitting thread. Production implementations -/// () persist to a local SQLite queue +/// () persist to the node's local SQLite queue /// first, then drain asynchronously to the actual historian. Per the Phase 7 plan, /// failed downstream writes replay with exponential backoff; /// operator actions are never blocked waiting on the historian. @@ -79,6 +79,18 @@ public enum HistorianDrainState Idle, Draining, BackingOff, + + /// + /// Ticking, but not draining: this node does not hold the Primary role, so it leaves the + /// (replicated) queue for the node that does. + /// + /// + /// Distinct from so a rising queue depth on a Secondary reads as the + /// designed behaviour rather than a stalled drain. It is also how a misconfigured pair + /// becomes diagnosable: if BOTH nodes report this, no one is draining and alarm history is + /// silently accumulating toward the capacity ceiling. + /// + NotPrimary, } /// Returned by the historian alarm writer per event — drain worker uses this to decide retry cadence. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/SqliteStoreAndForwardSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs similarity index 57% rename from src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/SqliteStoreAndForwardSink.cs rename to src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs index a76bd1ee..8c696694 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/SqliteStoreAndForwardSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs @@ -1,53 +1,54 @@ using System.Text.Json; -using Microsoft.Data.Sqlite; using Serilog; +using ZB.MOM.WW.LocalDb; namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; /// -/// Durable SQLite queue on the node absorbs every qualifying alarm event, a drain -/// worker batches rows to the Wonderware historian sidecar via -/// on an exponential-backoff cadence, and -/// operator acks never block on the historian being reachable. +/// Durable queue in the node's consolidated LocalDb absorbs every qualifying alarm event, a +/// drain worker batches rows to the historian gateway via +/// on an exponential-backoff cadence, and operator acks never block on the historian being +/// reachable. /// /// /// -/// Queue schema: -/// -/// CREATE TABLE Queue ( -/// RowId INTEGER PRIMARY KEY AUTOINCREMENT, -/// AlarmId TEXT NOT NULL, -/// EnqueuedUtc TEXT NOT NULL, -/// PayloadJson TEXT NOT NULL, -/// AttemptCount INTEGER NOT NULL DEFAULT 0, -/// LastAttemptUtc TEXT NULL, -/// LastError TEXT NULL, -/// DeadLettered INTEGER NOT NULL DEFAULT 0 -/// ); -/// -/// Dead-lettered rows stay in place for the configured retention window (default -/// 30 days) so operators can inspect + manually -/// retry before the sweeper purges them. Regular queue capacity is bounded — -/// overflow evicts the oldest non-dead-lettered rows with a WARN log. The -/// durability guarantee is therefore bounded by : -/// under a sustained historian outage, accepted events may be evicted before -/// delivery. The counter makes -/// overflow visible to operators without requiring the WARN log to be scraped. +/// Rows live in alarm_sf_events (see ), which the host +/// registers for replication. That is the point of this type living on +/// rather than owning its own file: the buffer mirrors to the +/// redundant pair peer, so a node that dies holding undelivered alarm history no longer +/// takes it to the grave. +/// +/// +/// Only one node of a pair may drain. Replication puts the Primary's queued rows in +/// the Secondary's table too; an ungated drain there would re-deliver every event, +/// continuously. The drainGate constructor argument is how the caller scopes +/// draining to the Primary. Enqueue is gated separately and upstream, by +/// HistorianAdapterActor. +/// +/// +/// Dead-lettered rows stay in place for the configured retention window (default 30 days) +/// so operators can inspect + manually retry before the sweeper purges them. Regular queue +/// capacity is bounded — overflow evicts the oldest non-dead-lettered rows with a WARN log. +/// The durability guarantee is therefore bounded by : under a +/// sustained historian outage, accepted events may be evicted before delivery. The +/// counter makes overflow visible to +/// operators without requiring the WARN log to be scraped. /// /// /// Drain runs on a self-rescheduling one-shot . /// Exponential backoff on : -/// 1s → 2s → 5s → 15s → 60s cap — the backoff is applied to the timer's next -/// due-time, so a historian outage genuinely slows the drain cadence. -/// rows flip -/// the DeadLettered flag on the individual row; neighbors in the batch -/// still retry on their own cadence. +/// 1s → 2s → 5s → 15s → 60s cap — the backoff is applied to the timer's next due-time, so a +/// historian outage genuinely slows the drain cadence. +/// rows flip the dead_lettered flag +/// on the individual row; neighbors in the batch still retry on their own cadence. /// /// -public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable +public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposable { /// Default queue capacity — oldest non-dead-lettered rows evicted past this. public const long DefaultCapacity = 1_000_000; + + /// Default window dead-lettered rows are retained for before the sweeper purges them. public static readonly TimeSpan DefaultDeadLetterRetention = TimeSpan.FromDays(30); /// Default max delivery attempts before a perpetually-retrying (poison) row is dead-lettered. @@ -62,7 +63,7 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable TimeSpan.FromSeconds(60), ]; - private readonly string _connectionString; + private readonly ILocalDb _db; private readonly IAlarmHistorianWriter _writer; private readonly ILogger _logger; private readonly int _batchSize; @@ -70,8 +71,9 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable private readonly TimeSpan _deadLetterRetention; private readonly int _maxAttempts; private readonly Func _clock; + private readonly Func _drainGate; - private readonly SemaphoreSlim _drainGate = new(1, 1); + private readonly SemaphoreSlim _drainGateLock = new(1, 1); private Timer? _drainTimer; private TimeSpan _tickInterval; private volatile int _backoffIndex; @@ -85,13 +87,16 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable private string? _lastError; private HistorianDrainState _drainState = HistorianDrainState.Idle; private long _evictedCount; + // Tracks the last gate decision so a denial is logged on the transition rather than on every + // tick. Null until the first drain runs. + private bool? _lastGateDecision; private long _queuedRowCount; // Probe counter — incremented every time we actually issue a real COUNT(*) for // capacity enforcement. Public for test instrumentation only. private long _capacityProbeCount; // After every Nth enqueue we resync the in-memory counter from storage to defend - // against silent drift (e.g. an external process editing the DB). + // against silent drift (e.g. the peer replicating rows in underneath us). private const long ResyncEnqueueInterval = 10_000; private long _enqueuesSinceResync; @@ -99,9 +104,13 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable public long DebugCapacityProbeCount => Interlocked.Read(ref _capacityProbeCount); /// - /// Initializes a new instance of the class with the specified configuration. + /// Initializes a new instance of the class. /// - /// The filesystem path to the SQLite database file. + /// + /// The node's local database. Its alarm_sf_events table must already exist and be + /// registered for replication — the host does both in LocalDbSetup.OnReady, before + /// any consumer can resolve this sink. + /// /// The alarm historian writer to handle batch forwarding. /// The logger for diagnostic output. /// The maximum number of rows to forward in a single batch. Defaults to 100. @@ -109,18 +118,24 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable /// The timespan to retain dead-lettered rows before purging. Defaults to 30 days. /// The maximum number of delivery attempts before a perpetually-retrying (poison) row is dead-lettered. Defaults to 10. /// Optional clock function for testing; defaults to . - public SqliteStoreAndForwardSink( - string databasePath, + /// + /// Consulted at the top of every drain tick; false skips the tick entirely, leaving + /// the queue intact. Callers running a redundant pair MUST supply a gate that is true on at + /// most one node, because the queue replicates — see the class remarks. The default + /// (null ⇒ always drain) is the correct single-node and test posture. + /// + public LocalDbStoreAndForwardSink( + ILocalDb db, IAlarmHistorianWriter writer, ILogger logger, int batchSize = 100, long capacity = DefaultCapacity, TimeSpan? deadLetterRetention = null, int maxAttempts = DefaultMaxAttempts, - Func? clock = null) + Func? clock = null, + Func? drainGate = null) { - if (string.IsNullOrWhiteSpace(databasePath)) - throw new ArgumentException("Database path required.", nameof(databasePath)); + _db = db ?? throw new ArgumentNullException(nameof(db)); _writer = writer ?? throw new ArgumentNullException(nameof(writer)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _batchSize = batchSize > 0 ? batchSize : throw new ArgumentOutOfRangeException(nameof(batchSize)); @@ -128,50 +143,11 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable _deadLetterRetention = deadLetterRetention ?? DefaultDeadLetterRetention; _maxAttempts = maxAttempts > 0 ? maxAttempts : throw new ArgumentOutOfRangeException(nameof(maxAttempts)); _clock = clock ?? (() => DateTime.UtcNow); - // DefaultTimeout gives ADO.NET command-level retry; the PRAGMA busy_timeout - // applied in OpenConnection backs it with SQLite's own busy-handler so an - // enqueue/drain collision waits out the file lock instead of throwing - // SQLITE_BUSY immediately. - _connectionString = new SqliteConnectionStringBuilder - { - DataSource = databasePath, - DefaultTimeout = 5, - }.ToString(); + _drainGate = drainGate ?? (static () => true); - InitializeSchema(); _queuedRowCount = ProbeQueuedRowCount(); } - /// - /// Open a connection with the busy timeout + WAL journal applied. SQLite - /// serializes writers with a file lock; the busy_timeout lets a writer wait - /// out a competing lock (default is 0 — fail fast), and WAL lets readers and - /// the single writer proceed without blocking each other. - /// - private SqliteConnection OpenConnection() - { - var conn = new SqliteConnection(_connectionString); - conn.Open(); - ApplyPragmas(conn); - return conn; - } - - /// Apply busy_timeout + WAL pragmas to an already-open connection (sync). - private static void ApplyPragmas(SqliteConnection conn) - { - using var pragma = conn.CreateCommand(); - pragma.CommandText = "PRAGMA busy_timeout=5000; PRAGMA journal_mode=WAL;"; - pragma.ExecuteNonQuery(); - } - - /// Apply busy_timeout + WAL pragmas to an already-open connection (async). - private static async Task ApplyPragmasAsync(SqliteConnection conn, CancellationToken ct) - { - using var pragma = conn.CreateCommand(); - pragma.CommandText = "PRAGMA busy_timeout=5000; PRAGMA journal_mode=WAL;"; - await pragma.ExecuteNonQueryAsync(ct).ConfigureAwait(false); - } - /// /// Start the background drain worker. Not started automatically so tests can /// drive deterministically. @@ -188,7 +164,7 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable /// The base interval between drain attempts. public void StartDrainLoop(TimeSpan tickInterval) { - if (_disposed) throw new ObjectDisposedException(nameof(SqliteStoreAndForwardSink)); + if (_disposed) throw new ObjectDisposedException(nameof(LocalDbStoreAndForwardSink)); _tickInterval = tickInterval; _drainTimer?.Dispose(); // One-shot: dueTime = tickInterval, period = Infinite. RescheduleDrain re-arms @@ -239,25 +215,31 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable public async Task EnqueueAsync(AlarmHistorianEvent evt, CancellationToken cancellationToken) { if (evt is null) throw new ArgumentNullException(nameof(evt)); - if (_disposed) throw new ObjectDisposedException(nameof(SqliteStoreAndForwardSink)); + if (_disposed) throw new ObjectDisposedException(nameof(LocalDbStoreAndForwardSink)); - using var conn = new SqliteConnection(_connectionString); - await conn.OpenAsync(cancellationToken).ConfigureAwait(false); - await ApplyPragmasAsync(conn, cancellationToken).ConfigureAwait(false); + await EnforceCapacityFastPathAsync(cancellationToken).ConfigureAwait(false); - await EnforceCapacityFastPathAsync(conn, cancellationToken).ConfigureAwait(false); + var payload = JsonSerializer.Serialize(evt); - using var cmd = conn.CreateCommand(); - cmd.CommandText = """ - INSERT INTO Queue (AlarmId, EnqueuedUtc, PayloadJson, AttemptCount) - VALUES ($alarmId, $enqueued, $payload, 0); - """; - cmd.Parameters.AddWithValue("$alarmId", evt.AlarmId); - cmd.Parameters.AddWithValue("$enqueued", _clock().ToString("O")); - cmd.Parameters.AddWithValue("$payload", JsonSerializer.Serialize(evt)); - await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + // ON CONFLICT DO NOTHING: re-enqueuing an identical event (the pair's boot-window + // double-accept) must not resurrect a row the drain already bumped or dead-lettered. + var inserted = await _db.ExecuteAsync( + """ + INSERT INTO alarm_sf_events + (id, alarm_id, enqueued_at_utc, payload_json, attempt_count) + VALUES (@Id, @AlarmId, @EnqueuedAtUtc, @PayloadJson, 0) + ON CONFLICT(id) DO NOTHING + """, + new + { + Id = AlarmSfSchema.DeriveId(payload), + AlarmId = evt.AlarmId, + EnqueuedAtUtc = _clock().ToString("O"), + PayloadJson = payload, + }, + cancellationToken).ConfigureAwait(false); - Interlocked.Increment(ref _queuedRowCount); + if (inserted > 0) Interlocked.Increment(ref _queuedRowCount); } /// @@ -268,15 +250,17 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable /// through which still runs a precise /// COUNT to compute the exact number of rows to evict. /// - private async Task EnforceCapacityFastPathAsync(SqliteConnection conn, CancellationToken ct) + private async Task EnforceCapacityFastPathAsync(CancellationToken ct) { var enqueuesSinceResync = Interlocked.Increment(ref _enqueuesSinceResync); var cached = Interlocked.Read(ref _queuedRowCount); - // Periodic resync — bounded amount of drift even under exotic conditions. + // Periodic resync — bounded amount of drift even under exotic conditions. Now doubly + // warranted: replication applies the peer's rows straight into the table, so the counter + // has a second way to fall behind that no local code path can observe. if (enqueuesSinceResync >= ResyncEnqueueInterval) { - await ResyncQueuedRowCountAsync(conn, ct).ConfigureAwait(false); + await ResyncQueuedRowCountAsync(ct).ConfigureAwait(false); cached = Interlocked.Read(ref _queuedRowCount); Interlocked.Exchange(ref _enqueuesSinceResync, 0); } @@ -286,55 +270,70 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable // Cached counter says we're at or above the capacity wall — fall back to the // precise path which probes COUNT(*) and evicts whatever's needed. - await EnforceCapacityAsync(conn, ct).ConfigureAwait(false); + await EnforceCapacityAsync(ct).ConfigureAwait(false); } /// Synchronously query COUNT(*) of non-dead-lettered rows. Used at startup. private long ProbeQueuedRowCount() { Interlocked.Increment(ref _capacityProbeCount); - using var conn = OpenConnection(); + using var conn = _db.CreateConnection(); using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM Queue WHERE DeadLettered = 0"; + cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0"; return (long)(cmd.ExecuteScalar() ?? 0L); } /// Re-sync the in-memory counter from storage (async path). - private async Task ResyncQueuedRowCountAsync(SqliteConnection conn, CancellationToken ct) + private async Task ResyncQueuedRowCountAsync(CancellationToken ct) { Interlocked.Increment(ref _capacityProbeCount); - using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM Queue WHERE DeadLettered = 0"; - var live = (long)(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) ?? 0L); + var live = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false); Interlocked.Exchange(ref _queuedRowCount, live); } + private async Task CountAsync(string predicate, CancellationToken ct) + { + var rows = await _db.QueryAsync( + $"SELECT COUNT(*) FROM alarm_sf_events WHERE {predicate}", + r => r.GetInt64(0), + parameters: null, + ct).ConfigureAwait(false); + return rows.Count > 0 ? rows[0] : 0L; + } + /// /// Read up to queued rows, forward through the writer, /// remove Ack'd rows, dead-letter PermanentFail rows, and extend the backoff /// on RetryPlease. Safe to call from multiple threads; the semaphore enforces /// serial execution. /// + /// + /// Returns without touching the queue when the drain gate is closed. On a redundant pair + /// that is the Secondary's steady state: its table fills by replication and stays put until + /// it is promoted. + /// /// Cancellation token for the operation. /// A task that represents the asynchronous operation. public async Task DrainOnceAsync(CancellationToken ct) { if (_disposed) return; - if (!await _drainGate.WaitAsync(0, ct).ConfigureAwait(false)) return; + if (!await _drainGateLock.WaitAsync(0, ct).ConfigureAwait(false)) return; try { + if (!EvaluateDrainGate()) + { + lock (_statusLock) { _drainState = HistorianDrainState.NotPrimary; } + return; + } + lock (_statusLock) { _drainState = HistorianDrainState.Draining; _lastDrainUtc = _clock(); } - // One connection per drain tick — used by purge, read, corrupt-dead-letter, - // and the outcome-applying transaction. - using var conn = OpenConnection(); - - PurgeAgedDeadLetters(conn); - var batch = ReadBatch(conn); + await PurgeAgedDeadLettersAsync(ct).ConfigureAwait(false); + var batch = await ReadBatchAsync(ct).ConfigureAwait(false); if (batch.Count == 0) { lock (_statusLock) { _drainState = HistorianDrainState.Idle; } @@ -342,22 +341,24 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable } // A null/un-deserializable payload can never succeed — dead-letter it - // immediately for its own RowId so it cannot stall the queue head, and + // immediately for its own id so it cannot stall the queue head, and // exclude it from the batch handed to the writer. - var corruptRowIds = batch.Where(r => r.Event is null).Select(r => r.RowId).ToList(); + var corruptIds = batch.Where(r => r.Event is null).Select(r => r.Id).ToList(); var liveRows = batch.Where(r => r.Event is not null).ToList(); var events = liveRows.Select(r => r.Event!).ToList(); - if (corruptRowIds.Count > 0) + if (corruptIds.Count > 0) { - using var corruptTx = conn.BeginTransaction(); - foreach (var rowId in corruptRowIds) - DeadLetterRow(conn, corruptTx, rowId, $"corrupt payload at {_clock():O}"); - corruptTx.Commit(); - Interlocked.Add(ref _queuedRowCount, -corruptRowIds.Count); + await using (var corruptTx = await _db.BeginTransactionAsync(ct).ConfigureAwait(false)) + { + foreach (var id in corruptIds) + await DeadLetterRowAsync(corruptTx, id, $"corrupt payload at {_clock():O}", ct).ConfigureAwait(false); + await corruptTx.CommitAsync(ct).ConfigureAwait(false); + } + Interlocked.Add(ref _queuedRowCount, -corruptIds.Count); _logger.Warning( "Dead-lettered {Count} historian queue row(s) with un-deserializable payload", - corruptRowIds.Count); + corruptIds.Count); } if (events.Count == 0) @@ -404,41 +405,44 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable } int rowsLeavingQueue = 0; - using (var tx = conn.BeginTransaction()) + await using (var tx = await _db.BeginTransactionAsync(ct).ConfigureAwait(false)) { for (var i = 0; i < outcomes.Count; i++) { var outcome = outcomes[i]; - var rowId = liveRows[i].RowId; + var id = liveRows[i].Id; switch (outcome) { case HistorianWriteOutcome.Ack: - DeleteRow(conn, tx, rowId); + // Deleted, not marked delivered. The replication engine carries the + // delete as a tombstone, so the peer drops its copy and cannot + // re-deliver the row after a failover. + await DeleteRowAsync(tx, id, ct).ConfigureAwait(false); rowsLeavingQueue++; break; case HistorianWriteOutcome.PermanentFail: - DeadLetterRow(conn, tx, rowId, $"permanent fail at {_clock():O}"); + await DeadLetterRowAsync(tx, id, $"permanent fail at {_clock():O}", ct).ConfigureAwait(false); rowsLeavingQueue++; break; case HistorianWriteOutcome.RetryPlease: // finding 002: cap retries so a perpetually-RetryPlease (poison) // row cannot retry forever at the 60s backoff floor. The incoming - // AttemptCount is the count BEFORE this attempt; +1 accounts for the + // attempt count is the count BEFORE this attempt; +1 accounts for the // bump this drain represents. At the cap, dead-letter instead of // bumping — and count it as leaving the live queue like PermanentFail. if (liveRows[i].AttemptCount + 1 >= _maxAttempts) { - DeadLetterRow(conn, tx, rowId, $"max attempts ({_maxAttempts}) exceeded"); + await DeadLetterRowAsync(tx, id, $"max attempts ({_maxAttempts}) exceeded", ct).ConfigureAwait(false); rowsLeavingQueue++; } else { - BumpAttempt(conn, tx, rowId, "retry-please"); + await BumpAttemptAsync(tx, id, "retry-please", ct).ConfigureAwait(false); } break; } } - tx.Commit(); + await tx.CommitAsync(ct).ConfigureAwait(false); } // Ack-deleted + PermanentFail-dead-lettered rows both leave the // non-dead-lettered queue, as do RetryPlease rows that hit the max-attempts @@ -464,22 +468,66 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable } finally { - _drainGate.Release(); + _drainGateLock.Release(); } } + /// + /// Consults the injected gate, logging only when the decision changes. + /// + /// + /// Logged at all because the closed state is indistinguishable from a healthy idle queue + /// from the outside, and one way to reach it is a redundancy snapshot that never names this + /// node — the documented identity-mismatch shape. Without this line, that misconfiguration + /// would present as alarm history quietly ceasing on both nodes of a pair. Logged on the + /// transition rather than per tick because the drain runs every few seconds. + /// + /// true when this tick may proceed. + private bool EvaluateDrainGate() + { + bool open; + try + { + open = _drainGate(); + } + catch (Exception ex) + { + // A throwing gate must not wedge the queue silently, and must not be treated as + // permission either: skip this tick, say why, and re-ask on the next one. + _logger.Error(ex, "Historian drain gate threw; skipping this tick"); + return false; + } + + bool? previous; + lock (_statusLock) + { + previous = _lastGateDecision; + _lastGateDecision = open; + } + + if (previous == open) return open; + + if (open) + _logger.Information("Historian drain resumed — this node now services the alarm queue"); + else + _logger.Information( + "Historian drain suspended — this node is not the Primary. Queued alarm events are retained and will drain from whichever node holds the Primary role"); + + return open; + } + /// public HistorianSinkStatus GetStatus() { - if (_disposed) throw new ObjectDisposedException(nameof(SqliteStoreAndForwardSink)); + if (_disposed) throw new ObjectDisposedException(nameof(LocalDbStoreAndForwardSink)); var queued = Interlocked.Read(ref _queuedRowCount); if (queued < 0) queued = 0; long deadlettered; - using (var conn = OpenConnection()) + using (var conn = _db.CreateConnection()) using (var cmd = conn.CreateCommand()) { - cmd.CommandText = "SELECT COUNT(*) FROM Queue WHERE DeadLettered = 1"; + cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1"; deadlettered = (long)(cmd.ExecuteScalar() ?? 0L); } @@ -510,10 +558,13 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable /// The number of rows revived from the dead-letter state. public int RetryDeadLettered() { - if (_disposed) throw new ObjectDisposedException(nameof(SqliteStoreAndForwardSink)); - using var conn = OpenConnection(); + if (_disposed) throw new ObjectDisposedException(nameof(LocalDbStoreAndForwardSink)); + // Through a LocalDb connection, not a private one: the capture triggers are what make this + // revival replicate to the peer, and they only exist on the registered table. + using var conn = _db.CreateConnection(); using var cmd = conn.CreateCommand(); - cmd.CommandText = "UPDATE Queue SET DeadLettered = 0, AttemptCount = 0, LastError = NULL WHERE DeadLettered = 1"; + cmd.CommandText = + "UPDATE alarm_sf_events SET dead_lettered = 0, attempt_count = 0, last_error = NULL WHERE dead_lettered = 1"; var revived = cmd.ExecuteNonQuery(); // Dead-lettered rows rejoin the non-dead-lettered queue — keep the in-memory // counter aligned. @@ -523,29 +574,31 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable /// /// One queued row paired with its deserialized event. is - /// null when the row's PayloadJson is corrupt or un-deserializable — - /// the always stays bound to its own row so outcomes can + /// null when the row's payload_json is corrupt or un-deserializable — + /// the always stays bound to its own row so outcomes can /// never be mapped to the wrong row. /// - private readonly record struct QueueRow(long RowId, AlarmHistorianEvent? Event, long AttemptCount); + private readonly record struct QueueRow(string Id, AlarmHistorianEvent? Event, long AttemptCount); - private List ReadBatch(SqliteConnection conn) + private async Task> ReadBatchAsync(CancellationToken ct) { - var rows = new List(); - using var cmd = conn.CreateCommand(); - cmd.CommandText = """ - SELECT RowId, PayloadJson, AttemptCount FROM Queue - WHERE DeadLettered = 0 - ORDER BY RowId ASC - LIMIT $limit - """; - cmd.Parameters.AddWithValue("$limit", _batchSize); - using var reader = cmd.ExecuteReader(); - while (reader.Read()) + // Ordered by enqueue time rather than insertion order: a hashed TEXT primary key carries + // no sequence the way the legacy AUTOINCREMENT rowid did. Round-trip ("O") timestamps sort + // lexicographically in chronological order; id makes the ordering total. + var raw = await _db.QueryAsync( + """ + SELECT id, payload_json, attempt_count FROM alarm_sf_events + WHERE dead_lettered = 0 + ORDER BY enqueued_at_utc ASC, id ASC + LIMIT @Limit + """, + r => (Id: r.GetString(0), Payload: r.GetString(1), AttemptCount: r.GetInt64(2)), + new { Limit = _batchSize }, + ct).ConfigureAwait(false); + + var rows = new List(raw.Count); + foreach (var (id, payload, attemptCount) in raw) { - var rowId = reader.GetInt64(0); - var payload = reader.GetString(1); - var attemptCount = reader.GetInt64(2); AlarmHistorianEvent? evt; try { @@ -556,77 +609,58 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable // Malformed JSON — carry a null event so the caller dead-letters this row. evt = null; } - rows.Add(new QueueRow(rowId, evt, attemptCount)); + rows.Add(new QueueRow(id, evt, attemptCount)); } return rows; } - private static void DeleteRow(SqliteConnection conn, SqliteTransaction tx, long rowId) - { - using var cmd = conn.CreateCommand(); - cmd.Transaction = tx; - cmd.CommandText = "DELETE FROM Queue WHERE RowId = $id"; - cmd.Parameters.AddWithValue("$id", rowId); - cmd.ExecuteNonQuery(); - } + private static Task DeleteRowAsync(ILocalDbTransaction tx, string id, CancellationToken ct) => + tx.ExecuteAsync("DELETE FROM alarm_sf_events WHERE id = @Id", new { Id = id }, ct); - private void DeadLetterRow(SqliteConnection conn, SqliteTransaction tx, long rowId, string reason) - { - using var cmd = conn.CreateCommand(); - cmd.Transaction = tx; - cmd.CommandText = """ - UPDATE Queue SET DeadLettered = 1, LastAttemptUtc = $now, LastError = $err, AttemptCount = AttemptCount + 1 - WHERE RowId = $id - """; - cmd.Parameters.AddWithValue("$now", _clock().ToString("O")); - cmd.Parameters.AddWithValue("$err", reason); - cmd.Parameters.AddWithValue("$id", rowId); - cmd.ExecuteNonQuery(); - } + private Task DeadLetterRowAsync(ILocalDbTransaction tx, string id, string reason, CancellationToken ct) => + tx.ExecuteAsync( + """ + UPDATE alarm_sf_events + SET dead_lettered = 1, last_attempt_utc = @Now, last_error = @Err, + attempt_count = attempt_count + 1 + WHERE id = @Id + """, + new { Now = _clock().ToString("O"), Err = reason, Id = id }, + ct); - private void BumpAttempt(SqliteConnection conn, SqliteTransaction tx, long rowId, string reason) - { - using var cmd = conn.CreateCommand(); - cmd.Transaction = tx; - cmd.CommandText = """ - UPDATE Queue SET LastAttemptUtc = $now, LastError = $err, AttemptCount = AttemptCount + 1 - WHERE RowId = $id - """; - cmd.Parameters.AddWithValue("$now", _clock().ToString("O")); - cmd.Parameters.AddWithValue("$err", reason); - cmd.Parameters.AddWithValue("$id", rowId); - cmd.ExecuteNonQuery(); - } + private Task BumpAttemptAsync(ILocalDbTransaction tx, string id, string reason, CancellationToken ct) => + tx.ExecuteAsync( + """ + UPDATE alarm_sf_events + SET last_attempt_utc = @Now, last_error = @Err, attempt_count = attempt_count + 1 + WHERE id = @Id + """, + new { Now = _clock().ToString("O"), Err = reason, Id = id }, + ct); - // Async variant used by EnqueueAsync. - private async Task EnforceCapacityAsync(SqliteConnection conn, CancellationToken ct) + private async Task EnforceCapacityAsync(CancellationToken ct) { Interlocked.Increment(ref _capacityProbeCount); - long count; - using (var cmd = conn.CreateCommand()) - { - cmd.CommandText = "SELECT COUNT(*) FROM Queue WHERE DeadLettered = 0"; - count = (long)(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) ?? 0L); - } + var count = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false); + // Resync the in-memory counter while we have a fresh number. Interlocked.Exchange(ref _queuedRowCount, count); if (count < _capacity) return; var toEvict = count - _capacity + 1; - using (var cmd = conn.CreateCommand()) - { - cmd.CommandText = """ - DELETE FROM Queue - WHERE RowId IN ( - SELECT RowId FROM Queue - WHERE DeadLettered = 0 - ORDER BY RowId ASC - LIMIT $n - ) - """; - cmd.Parameters.AddWithValue("$n", toEvict); - await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); - } + await _db.ExecuteAsync( + """ + DELETE FROM alarm_sf_events + WHERE id IN ( + SELECT id FROM alarm_sf_events + WHERE dead_lettered = 0 + ORDER BY enqueued_at_utc ASC, id ASC + LIMIT @N + ) + """, + new { N = toEvict }, + ct).ConfigureAwait(false); + Interlocked.Add(ref _queuedRowCount, -toEvict); long lifetimeEvicted; lock (_statusLock) { _evictedCount += toEvict; lifetimeEvicted = _evictedCount; } @@ -635,40 +669,21 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable _capacity, toEvict, lifetimeEvicted); } - private void PurgeAgedDeadLetters(SqliteConnection conn) + private async Task PurgeAgedDeadLettersAsync(CancellationToken ct) { var cutoff = (_clock() - _deadLetterRetention).ToString("O"); - using var cmd = conn.CreateCommand(); - cmd.CommandText = """ - DELETE FROM Queue - WHERE DeadLettered = 1 AND LastAttemptUtc IS NOT NULL AND LastAttemptUtc < $cutoff - """; - cmd.Parameters.AddWithValue("$cutoff", cutoff); - var purged = cmd.ExecuteNonQuery(); + var purged = await _db.ExecuteAsync( + """ + DELETE FROM alarm_sf_events + WHERE dead_lettered = 1 AND last_attempt_utc IS NOT NULL AND last_attempt_utc < @Cutoff + """, + new { Cutoff = cutoff }, + ct).ConfigureAwait(false); + if (purged > 0) _logger.Information("Purged {Count} dead-lettered row(s) past retention window", purged); } - private void InitializeSchema() - { - using var conn = OpenConnection(); - using var cmd = conn.CreateCommand(); - cmd.CommandText = """ - CREATE TABLE IF NOT EXISTS Queue ( - RowId INTEGER PRIMARY KEY AUTOINCREMENT, - AlarmId TEXT NOT NULL, - EnqueuedUtc TEXT NOT NULL, - PayloadJson TEXT NOT NULL, - AttemptCount INTEGER NOT NULL DEFAULT 0, - LastAttemptUtc TEXT NULL, - LastError TEXT NULL, - DeadLettered INTEGER NOT NULL DEFAULT 0 - ); - CREATE INDEX IF NOT EXISTS IX_Queue_Drain ON Queue (DeadLettered, RowId); - """; - cmd.ExecuteNonQuery(); - } - private void BumpBackoff() => _backoffIndex = Math.Min(_backoffIndex + 1, BackoffLadder.Length - 1); private void ResetBackoff() => _backoffIndex = 0; @@ -676,12 +691,16 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable public TimeSpan CurrentBackoff => BackoffLadder[_backoffIndex]; /// Disposes the sink and releases all held resources including the drain timer and the writer. + /// + /// The is NOT disposed here — it is a shared singleton owned by the + /// host and used by the deployment cache too. + /// public void Dispose() { if (_disposed) return; _disposed = true; _drainTimer?.Dispose(); - _drainGate.Dispose(); + _drainGateLock.Dispose(); if (_writer is IDisposable writerDisposable) writerDisposable.Dispose(); } } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj index 62515df4..7e225b8d 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj @@ -20,6 +20,12 @@ --> + + diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayAlarmHistorianWriter.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayAlarmHistorianWriter.cs index 02eb624a..1817c8a3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayAlarmHistorianWriter.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayAlarmHistorianWriter.cs @@ -9,7 +9,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway; /// /// backed by the HistorianGateway SendEvent path. The -/// drain worker behind SqliteStoreAndForwardSink calls +/// drain worker behind LocalDbStoreAndForwardSink calls /// and uses the returned per-event /// to decide retry vs. dead-letter, so this writer maps every /// gateway result — success ack, the published client's typed exception hierarchy, raw diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs index 6033bde7..4df51b33 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs @@ -47,7 +47,7 @@ public static class GatewayHistorian /// — the same single gateway the read path /// () targets. The Host's AddAlarmHistorian wiring supplies /// this as the concrete the durable - /// SqliteStoreAndForwardSink drain worker delegates to, sourcing the connection from the + /// LocalDbStoreAndForwardSink drain worker delegates to, sourcing the connection from the /// ServerHistorian section (endpoint/key/TLS) rather than the legacy Wonderware-shaped /// AlarmHistorian host/port. Resolves an and the writer's /// from , falling back to the null diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/HistorianGatewayClientAdapter.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/HistorianGatewayClientAdapter.cs index 5f429f30..c7a323d1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/HistorianGatewayClientAdapter.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/HistorianGatewayClientAdapter.cs @@ -57,16 +57,23 @@ public sealed class HistorianGatewayClientAdapter : IHistorianGatewayClient, IDi $"ServerHistorian:Endpoint must be an absolute http(s) URI (e.g. https://host:5222); got '{options.Endpoint}'."); } + // TLS-only options must stay at their defaults on a plaintext h2c connection: the client rejects + // each of them outright when UseTls=false (" is a TLS-only option and requires + // UseTls=true"). Forwarding them unconditionally made h2c UNREACHABLE — AllowUntrustedServer- + // Certificate defaults to false, so RequireCertificateValidation was always sent as true and + // every http:// deployment crashed at startup, even though the scheme is documented as the + // supported way to select h2c. There is no certificate to have a posture about here. var clientOptions = new HistorianGatewayClientOptions { Endpoint = endpointUri, ApiKey = options.ApiKey, UseTls = options.UseTls, - CaCertificatePath = options.CaCertificatePath, - // INVERTED mapping: ServerHistorianOptions.AllowUntrustedServerCertificate (opt-in to accept - // a self-signed cert) is the negation of the client's RequireCertificateValidation. Allowing - // an untrusted cert == not requiring validation; a pinned CaCertificatePath always verifies. - RequireCertificateValidation = !options.AllowUntrustedServerCertificate, + CaCertificatePath = options.UseTls ? options.CaCertificatePath : null, + // INVERTED mapping (TLS only): ServerHistorianOptions.AllowUntrustedServerCertificate + // (opt-in to accept a self-signed cert) is the negation of the client's + // RequireCertificateValidation. Allowing an untrusted cert == not requiring validation; a + // pinned CaCertificatePath always verifies. + RequireCertificateValidation = options.UseTls && !options.AllowUntrustedServerCertificate, DefaultCallTimeout = options.CallTimeout, LoggerFactory = loggerFactory, }; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/AlarmsHistorian.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/AlarmsHistorian.razor index db03988c..a957ca51 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/AlarmsHistorian.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/AlarmsHistorian.razor @@ -18,8 +18,9 @@
Snapshot from the local node's HistorianAdapterActor. Default sink is a no-op (NullAlarmHistorianSink); production wires - SqliteStoreAndForwardSink draining to the HistorianGateway - (SendEvent) behind it. Polling every @PollSeconds s. + LocalDbStoreAndForwardSink — buffering into this node's LocalDb, and + draining to the HistorianGateway (SendEvent) only while this node holds + the Primary role. Polling every @PollSeconds s.
@if (_status is null) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/AlarmSfLegacyMigrator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/AlarmSfLegacyMigrator.cs new file mode 100644 index 00000000..8e6cc31b --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/AlarmSfLegacyMigrator.cs @@ -0,0 +1,217 @@ +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Configuration; +using ZB.MOM.WW.LocalDb; +using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; + +namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; + +/// +/// One-time copy of the pre-consolidation alarm-historian.db store-and-forward queue +/// into the consolidated ZB.MOM.WW.LocalDb database. +/// +/// +/// +/// What is at stake. Rows sit in that file precisely because the historian could not +/// be reached. Discarding them on upgrade would throw away exactly the alarm audit trail +/// the store-and-forward queue exists to protect — so the copy tolerates an older column +/// set rather than bailing out, and refuses to rename a file it did not actually read. +/// +/// +/// Runs after RegisterReplicated, deliberately. Capture is trigger-based, so +/// rows inserted before registration never enter the oplog and never reach the peer — +/// silently, and permanently, because nothing revisits history. Migrating after +/// registration means the recovered backlog replicates like any other write. +/// +/// +/// Ids come from the payload, not from the legacy row id. The legacy primary key was +/// RowId INTEGER PRIMARY KEY AUTOINCREMENT, which cannot survive into a replicated +/// table: each node allocated its own sequence, so node A's row 7 and node B's row 7 are +/// different alarms that would silently overwrite one another under last-writer-wins. +/// Hashing the payload () solves that and one more +/// problem besides — a warm pair's two legacy files overlap, because +/// HistorianAdapterActor default-writes while its redundancy role is unknown and +/// both nodes therefore accepted the same transitions during every boot window. A +/// node-prefixed id would carry that duplication into the merged buffer forever; an +/// equal-payload id collapses it. It is also what makes a re-run harmless under +/// INSERT OR IGNORE. +/// +/// +/// All-or-nothing. The copy runs in one transaction and the legacy file is renamed +/// to <name>.migrated only after the commit. A failure throws out of +/// OnReady, failing host startup with the legacy file untouched — no half-migrated +/// state to reason about, and the operator still holds the original. A later boot sees the +/// renamed file and no-ops. +/// +/// +public static class AlarmSfLegacyMigrator +{ + private const string MigratedSuffix = ".migrated"; + + /// The legacy queue table. + private const string LegacyTable = "Queue"; + + /// + /// The configuration key that used to carry the standalone queue file's path. Read as a raw + /// key because the corresponding AlarmHistorianOptions property was removed with the + /// bespoke file management it configured. + /// + public const string LegacyPathKey = "AlarmHistorian:DatabasePath"; + + /// The pre-removal default for . + private const string DefaultLegacyPath = "alarm-historian.db"; + + /// + /// The legacy column whose absence means this is not a queue file we can read. Without the + /// payload there is no event and no id to derive from one. + /// + private const string RequiredColumn = "PayloadJson"; + + /// + /// Every legacy column, mapped to its consolidated counterpart. Columns absent from an + /// older file are dropped from the copy rather than failing it — see + /// . + /// + private static readonly (string Legacy, string Current)[] ColumnMap = + [ + ("AlarmId", "alarm_id"), + ("EnqueuedUtc", "enqueued_at_utc"), + ("PayloadJson", "payload_json"), + ("AttemptCount", "attempt_count"), + ("LastAttemptUtc", "last_attempt_utc"), + ("LastError", "last_error"), + ("DeadLettered", "dead_lettered"), + ]; + + /// + /// Copies any legacy alarm queue into , then renames the legacy file. + /// + /// The consolidated database, with alarm_sf_events already registered. + /// Configuration supplying the legacy queue's path. + public static void Migrate(ILocalDb db, IConfiguration config) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(config); + + var legacyPath = ResolveLegacyPath(config); + if (!ShouldMigrate(legacyPath)) return; + + using (var legacy = OpenLegacyReadOnly(legacyPath)) + { + var present = PresentColumns(legacy); + + // An absent table probes as zero columns, so this one guard covers both "no queue + // table" and "a shape we do not recognise". Returning without renaming leaves the file + // for an operator to inspect. + if (!present.Contains(RequiredColumn)) return; + + using var connection = db.CreateConnection(); + using var transaction = connection.BeginTransaction(); + CopyRows(legacy, connection, transaction, present); + transaction.Commit(); + } + + // Only after the commit. A crash between the two leaves the file in place and the next + // boot copies it again, which the payload-derived ids make harmless. + File.Move(legacyPath, legacyPath + MigratedSuffix, overwrite: true); + } + + /// + /// Resolves the legacy queue path from the removed configuration key, falling back to the + /// code default it used to carry. + /// + /// + /// Relative paths resolve against the process working directory — not the LocalDb + /// directory — because that is where the old code actually put them. + /// + /// The application configuration. + /// An absolute path, or empty when there is nothing durable to migrate from. + internal static string ResolveLegacyPath(IConfiguration config) + { + var path = config[LegacyPathKey] ?? DefaultLegacyPath; + + // In-memory and URI-form sources (test / dev configurations) have nothing durable behind + // them, and Path.GetFullPath on them would produce nonsense. + if (string.IsNullOrWhiteSpace(path) || + path.Equals(":memory:", StringComparison.OrdinalIgnoreCase) || + path.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + { + return string.Empty; + } + + return Path.GetFullPath(path); + } + + /// Whether there is a legacy file worth opening. + private static bool ShouldMigrate(string legacyPath) => + !string.IsNullOrEmpty(legacyPath) + && File.Exists(legacyPath) + && !File.Exists(legacyPath + MigratedSuffix); + + /// Opens the legacy file read-only, so a failed migration cannot damage it. + private static SqliteConnection OpenLegacyReadOnly(string path) + { + var connection = new SqliteConnection(new SqliteConnectionStringBuilder + { + DataSource = path, + Mode = SqliteOpenMode.ReadOnly, + }.ToString()); + connection.Open(); + return connection; + } + + /// + /// Which of the columns we know about the legacy table actually has. + /// + /// + /// A file written by an older build predates some columns, and naming a missing one in the + /// SELECT throws "no such column" — which discards every row in the table rather than the + /// one field. Intersecting first means an old file migrates its data and simply leaves the + /// newer columns at their schema defaults. + /// + private static HashSet PresentColumns(SqliteConnection legacy) + { + var present = new HashSet(StringComparer.OrdinalIgnoreCase); + + using var cmd = legacy.CreateCommand(); + cmd.CommandText = $"SELECT name FROM pragma_table_info('{LegacyTable}')"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + present.Add(reader.GetString(0)); + + return present; + } + + /// Copies every legacy row into the consolidated table, keyed by payload hash. + private static void CopyRows( + SqliteConnection legacy, + SqliteConnection target, + SqliteTransaction transaction, + HashSet present) + { + var columns = ColumnMap.Where(c => present.Contains(c.Legacy)).ToArray(); + var payloadOrdinal = Array.FindIndex(columns, c => c.Legacy == RequiredColumn); + + using var read = legacy.CreateCommand(); + read.CommandText = + $"SELECT {string.Join(", ", columns.Select(c => c.Legacy))} FROM {LegacyTable}"; + + using var reader = read.ExecuteReader(); + while (reader.Read()) + { + using var insert = target.CreateCommand(); + insert.Transaction = transaction; + insert.CommandText = $""" + INSERT OR IGNORE INTO {AlarmSfSchema.EventsTable} + ({AlarmSfSchema.IdColumn}, {string.Join(", ", columns.Select(c => c.Current))}) + VALUES + ($id, {string.Join(", ", columns.Select((_, i) => $"$c{i}"))}) + """; + + insert.Parameters.AddWithValue("$id", AlarmSfSchema.DeriveId(reader.GetString(payloadOrdinal))); + for (var i = 0; i < columns.Length; i++) + insert.Parameters.AddWithValue($"$c{i}", reader.GetValue(i) ?? DBNull.Value); + + insert.ExecuteNonQuery(); + } + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbRegistration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbRegistration.cs index c23fa6bc..aa7522ec 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbRegistration.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbRegistration.cs @@ -88,7 +88,7 @@ public static class LocalDbRegistration ArgumentNullException.ThrowIfNull(services); ArgumentNullException.ThrowIfNull(configuration); - services.AddZbLocalDb(configuration, LocalDbSetup.OnReady); + services.AddZbLocalDb(configuration, db => LocalDbSetup.OnReady(db, configuration)); services.AddZbLocalDbReplication(configuration); // The consumer-facing seam. WithOtOpcUaRuntimeActors resolves this optionally and threads it diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs index 0ce2d3be..ecbd6c8a 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs @@ -1,11 +1,13 @@ +using Microsoft.Extensions.Configuration; using ZB.MOM.WW.LocalDb; +using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache; namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; /// -/// The onReady callback handed to AddZbLocalDb: creates the deployment-cache -/// tables and opts them into replication. +/// The onReady callback handed to AddZbLocalDb: creates the deployment-cache and +/// alarm store-and-forward tables and opts them into replication. /// /// /// @@ -26,23 +28,41 @@ public static class LocalDbSetup /// RegisterReplicated is what installs the three AFTER triggers that capture /// changes into the oplog. Any row written before that call is never captured, so it /// never reaches the peer — silently, and permanently, because nothing ever revisits - /// history. Phase 1 writes nothing here; when Phase 2 adds its store-and-forward - /// migrator, the migrator must run after both registrations for the same reason. + /// history. therefore runs last, after every + /// registration — it writes rows, and they must be captured like any other write. + /// + /// + /// The alarm buffer's tables are created unconditionally, regardless of whether this + /// node has AlarmHistorian:Enabled set. An empty registered table costs three + /// triggers and nothing else, whereas creating it lazily when the sink first appears + /// would mean a node that enables the historian later writes rows before its triggers + /// exist — which is precisely the silent-loss shape above. /// /// /// The freshly constructed local database. - public static void OnReady(ILocalDb db) + /// + /// Application configuration, read by the legacy migrator for the pre-consolidation queue's + /// path. Required rather than optional: an overload that silently skipped the migration + /// would be one wiring mistake away from discarding a node's undelivered alarm history. + /// + public static void OnReady(ILocalDb db, IConfiguration configuration) { ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(configuration); // CreateConnection() hands back an already-open, pragma-configured connection carrying the // zb_hlc_next() UDF the capture triggers need. Calling Open() on it would throw. using (var connection = db.CreateConnection()) { DeploymentCacheSchema.Apply(connection); + AlarmSfSchema.Apply(connection); } db.RegisterReplicated(DeploymentCacheSchema.ArtifactsTable); db.RegisterReplicated(DeploymentCacheSchema.PointerTable); + db.RegisterReplicated(AlarmSfSchema.EventsTable); + + // LAST, and only here. This is the one call in OnReady that writes rows. + AlarmSfLegacyMigrator.Migrate(db, configuration); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs index ca727a5f..a9ebcc86 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs @@ -24,12 +24,25 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; /// in the Host, so Enabled covers it. /// /// -/// Fail tier = provably-crashing configs only. Only an empty / non-absolute / non-http(s) -/// Endpoint fails here (it throws in the factory). Empty ApiKey and non-positive -/// MaxTieClusterOverfetch degrade rather than crash (the gateway rejects calls / the node -/// manager surfaces a Bad read), so they stay operator warnings in +/// Fail tier = provably-crashing configs only. Three settings qualify, and they all fail +/// the same way — HistorianGatewayClientOptions.Validate() throws inside the client +/// factory before any call is attempted, so the host dies at startup: +/// an empty / non-absolute / non-http(s) Endpoint; an empty ApiKey ("The gateway +/// API key must not be empty"); and a UseTls that disagrees with the endpoint scheme +/// ("UseTls requires an https gateway endpoint" / "An https gateway endpoint requires UseTls"). +/// A non-positive MaxTieClusterOverfetch genuinely degrades rather than crashes (the node +/// manager surfaces a Bad read), so it stays an operator warning in /// . The Endpoint value is not a secret and -/// is echoed to make the error actionable; the ApiKey is never surfaced. +/// is echoed to make each error actionable; the ApiKey is never surfaced. +/// +/// +/// The ApiKey and UseTls checks were added after two consecutive live-rig +/// crash-loops (LocalDb Phase 2 gate). Both had been classified as degrading, on the +/// assumption that a bad client configuration would connect and be rejected by the gateway — but +/// the client validates its own options at construction, so the process dies during Akka startup +/// instead, which is precisely the failure this validator exists to convert into a named, +/// aggregated error. The lesson generalises: "degrades" is only true of settings the client +/// does not itself validate. /// /// public sealed class ServerHistorianOptionsValidator : OptionsValidatorBase @@ -62,5 +75,26 @@ public sealed class ServerHistorianOptionsValidator : OptionsValidatorBase. private readonly Func? _driverMemberCountProvider; + /// Where the Primary-gate decision is published for consumers that live outside a mailbox — + /// today, the alarm store-and-forward drain. Null on nodes that do not wire one. + private readonly IRedundancyRoleView? _redundancyRoleView; + + /// + /// Host of this node's LocalDb replication partner — the only node that holds a copy of this + /// node's alarm queue, and therefore the only node it may ever stand down in favour of. + /// null when this node dials nobody (unpaired, or the listening half of a pair). + /// + private readonly string? _replicationPeerHost; + /// Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent /// identity mismatch (03/S5) would otherwise log on every snapshot. private bool _warnedSnapshotMissingLocalNode; @@ -366,7 +378,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers IActorRef? scriptedAlarmHostOverride = null, IDriverCapabilityInvokerFactory? invokerFactory = null, Func? driverMemberCountProvider = null, - IDeploymentArtifactCache? deploymentArtifactCache = null) => + IDeploymentArtifactCache? deploymentArtifactCache = null, + IRedundancyRoleView? redundancyRoleView = null, + string? replicationPeerHost = null) => // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an // expression tree. Six IActorRef? parameters and several interface-typed ones mean a // mis-ordered argument is usually type-compatible and therefore compiles clean, then binds @@ -376,7 +390,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor, healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride, loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider, - deploymentArtifactCache)); + deploymentArtifactCache, redundancyRoleView, replicationPeerHost)); /// Initializes a new DriverHostActor with the specified dependencies. /// Database context factory for configuration database access. @@ -426,9 +440,13 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers IActorRef? scriptedAlarmHostOverride = null, IDriverCapabilityInvokerFactory? invokerFactory = null, Func? driverMemberCountProvider = null, - IDeploymentArtifactCache? deploymentArtifactCache = null) + IDeploymentArtifactCache? deploymentArtifactCache = null, + IRedundancyRoleView? redundancyRoleView = null, + string? replicationPeerHost = null) { _deploymentArtifactCache = deploymentArtifactCache; + _redundancyRoleView = redundancyRoleView; + _replicationPeerHost = replicationPeerHost; _dbFactory = dbFactory; _localNode = localNode; _coordinatorOverride = coordinator; @@ -1457,6 +1475,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers } } + /// Host part of a host:port node id. + private static string HostOf(NodeId nodeId) + { + var text = nodeId.ToString(); + var colon = text.LastIndexOf(':'); + return colon < 0 ? text : text[..colon]; + } + /// The Primary-gate deny reason tag for the denial meter (secondary|detached|role-unknown). private string PrimaryGateDenyReason() => _localRole switch { @@ -1476,19 +1502,35 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers if (local is not null) { _localRole = local.Role; - return; } - - // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the - // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent - // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable. - if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1) + else if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1) { + // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the + // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent + // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable. _warnedSnapshotMissingLocalNode = true; _log.Warning( "DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster", _localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId))); } + + // Publish on EVERY snapshot, including ones that leave the cached role untouched, so a role + // that changes without this node being named still reaches the drain within one heartbeat. + // + // NOTE the different policy: the drain gate keys on the role ALONE and opens when it is + // unknown, where the data-plane gate above resolves an unknown role by member count and + // denies. Publishing ShouldServiceAsPrimary() here suspended the drain on every node of a + // multi-driver cluster that had no redundancy configured — see ShouldDrainAlarmHistory. + // Defer ONLY to the node that holds a copy of our rows. The redundancy election is + // cluster-wide and the queue is pair-local, so "somebody is Primary" is not a licence to stop + // draining — that Primary is usually in another pair entirely, and cannot deliver our events. + var peerIsPrimary = + _replicationPeerHost is not null + && msg.Nodes.Any(n => n.Role == RedundancyRole.Primary + && HostOf(n.NodeId) == _replicationPeerHost); + + _redundancyRoleView?.Publish( + PrimaryGatePolicy.ShouldDrainAlarmHistory(_localRole, queueSharingPeerIsPrimary: peerIsPrimary)); } private void Stale() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/PrimaryGatePolicy.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/PrimaryGatePolicy.cs index 90486a19..86a0e3bf 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/PrimaryGatePolicy.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/PrimaryGatePolicy.cs @@ -25,4 +25,50 @@ public static class PrimaryGatePolicy RedundancyRole.Secondary or RedundancyRole.Detached => false, _ => driverMemberCount <= 1, // unknown: allow only when no driver peer exists }; + + /// + /// Decide whether this node should drain the replicated alarm store-and-forward queue. + /// + /// + /// + /// Deliberately not . That gate protects a shared + /// field device, where two nodes acting at once is dangerous and irreversible, so an unknown + /// role with a peer present must deny. This gate protects an append-only historian, and the + /// delivery contract is already at-least-once across a failover, by design — identical + /// events even collapse to a single row, because ids hash the payload. + /// + /// + /// The two error costs are therefore asymmetric: a false allow costs a duplicate history row; + /// a false deny silently stops the alarm audit trail and eventually evicts it at the capacity + /// wall. So this decision keys on the role alone and opens when the role is unknown — + /// no membership tie-break, because cluster-wide driver count says nothing about whether + /// this node has a redundant partner. + /// + /// + /// Found by the LocalDb Phase 2 live gate: on a rig of four driver nodes in one cluster with + /// no Redundancy section, borrowing the write gate's verdict suspended the drain on + /// every node — including unpaired ones — so nothing drained anywhere. + /// + /// + /// This node's last-known redundancy role, or null when unknown. + /// + /// Whether the node holding in the same snapshot is a node + /// that shares this node's queue — i.e. its LocalDb replication partner. + /// + /// Merely knowing that a Primary exists is not enough, because the redundancy role is + /// a CLUSTER-WIDE election while the alarm queue is PAIR-LOCAL. A fleet runs many pairs in + /// one cluster, so the elected driver Primary is usually in some other pair — and on the + /// docker-dev rig it is a central node, which carries the driver Akka role, replicates + /// nobody's LocalDb and does not even run the alarm historian. Deferring to it means this + /// node's events are delivered by no one, ever. + /// + /// + /// true to drain; false only when a node holding these same rows is doing it. + public static bool ShouldDrainAlarmHistory(RedundancyRole? localRole, bool queueSharingPeerIsPrimary) => + localRole switch + { + RedundancyRole.Primary => true, + RedundancyRole.Secondary or RedundancyRole.Detached => !queueSharingPeerIsPrimary, + _ => true, // unknown role ⇒ drain; a duplicate row beats a silent gap + }; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/AlarmHistorianOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/AlarmHistorianOptions.cs index 6bced110..9d7ed4a7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/AlarmHistorianOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/AlarmHistorianOptions.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using System.IO; using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian; @@ -7,11 +6,12 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian; /// /// Binds the AlarmHistorian configuration section that gates the durable /// store-and-forward alarm sink. When is true, -/// AddAlarmHistorian registers a SqliteStoreAndForwardSink (draining to the +/// AddAlarmHistorian registers a LocalDbStoreAndForwardSink (draining to the /// gateway alarm writer supplied by the Host) in place of the /// NullAlarmHistorianSink default; otherwise the Null default survives. This section -/// supplies only the gate and the SQLite store-and-forward knobs — the -/// downstream connection (endpoint/key/TLS) is sourced from the ServerHistorian section. +/// supplies only the gate and the store-and-forward knobs — the +/// downstream connection (endpoint/key/TLS) is sourced from the ServerHistorian section, +/// and the queue's storage location is the node's consolidated LocalDb:Path database. /// public sealed class AlarmHistorianOptions { @@ -24,9 +24,6 @@ public sealed class AlarmHistorianOptions /// public bool Enabled { get; init; } - /// Filesystem path to the local SQLite store-and-forward queue database. - public string DatabasePath { get; init; } = "alarm-historian.db"; - /// Maximum number of queued rows the drain worker forwards in a single batch. public int BatchSize { get; init; } = 100; @@ -34,15 +31,15 @@ public sealed class AlarmHistorianOptions public int DrainIntervalSeconds { get; init; } = 5; /// Maximum queued rows before the sink evicts the oldest. Defaults to 1,000,000 - /// (matches SqliteStoreAndForwardSink's DefaultCapacity). - public long Capacity { get; init; } = SqliteStoreAndForwardSink.DefaultCapacity; + /// (matches LocalDbStoreAndForwardSink's DefaultCapacity). + public long Capacity { get; init; } = LocalDbStoreAndForwardSink.DefaultCapacity; /// Days to retain dead-lettered rows before purge. Defaults to 30. public int DeadLetterRetentionDays { get; init; } = 30; /// Maximum delivery attempts before a perpetually-retrying (poison) row is dead-lettered. - /// Defaults to 10 (matches SqliteStoreAndForwardSink's DefaultMaxAttempts). - public int MaxAttempts { get; init; } = SqliteStoreAndForwardSink.DefaultMaxAttempts; + /// Defaults to 10 (matches LocalDbStoreAndForwardSink's DefaultMaxAttempts). + public int MaxAttempts { get; init; } = LocalDbStoreAndForwardSink.DefaultMaxAttempts; /// Returns operator-facing misconfiguration warnings for an Enabled historian /// (empty when disabled or correctly configured). Pure — the registration logs each entry. @@ -51,8 +48,6 @@ public sealed class AlarmHistorianOptions { var warnings = new List(); if (!Enabled) return warnings; - if (!Path.IsPathRooted(DatabasePath)) - warnings.Add($"AlarmHistorian:DatabasePath '{DatabasePath}' is relative — it resolves against the process working directory (e.g. System32 for a Windows service). Set an absolute path."); if (DrainIntervalSeconds <= 0) warnings.Add($"AlarmHistorian:DrainIntervalSeconds is {DrainIntervalSeconds} — must be > 0; the drain timer will throw or spin at startup."); if (Capacity <= 0) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs index 93eb4142..af60b86b 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs @@ -15,7 +15,7 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian; /// Thin actor wrapper around . Engine code (ScriptedAlarmActor, /// Galaxy native alarm bridge, AB CIP ALMD reader) tells s to this /// actor; the actor enqueues them on the sink fire-and-forget. Production deployments register -/// against IAlarmHistorianSink; the sink owns the +/// against IAlarmHistorianSink; the sink owns the /// durable queue + drain-to-HistorianGateway-SendEvent loop. The actor here owns nothing operational beyond /// the message contract — its job is to keep the engine actors on Akka's mailbox without blocking /// them on disk I/O or gateway round-trips. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs new file mode 100644 index 00000000..a9c41dff --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs @@ -0,0 +1,57 @@ +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; + +/// +/// A singleton snapshot of the alarm-history drain decision, so code that lives outside an actor +/// can ask "should this node be draining the alarm queue right now?". +/// +/// +/// +/// Exists for the alarm store-and-forward drain. That drain runs on a timer owned by the +/// sink, not on an actor mailbox, so it cannot receive RedundancyStateChanged +/// directly — but it must be role-scoped, because the queue it drains replicates to the +/// pair peer and two draining nodes would deliver every event twice. +/// +/// +/// Deliberately publishes the decision rather than the role that produced it, so +/// stays the single place that decides. +/// +/// +/// This is not the device-write gate. It is fed by +/// , which opens on an unknown role +/// where closes. The naming is +/// explicit about the consumer for that reason: an earlier revision published the +/// write-gate verdict here, and on a cluster with several driver nodes but no configured +/// redundancy it suspended the drain everywhere, so alarm history accumulated on every node +/// and left none. +/// +/// +public interface IRedundancyRoleView +{ + /// Whether this node should be draining the alarm store-and-forward queue right now. + bool ShouldDrainAlarmHistory { get; } + + /// Records a fresh decision. Called by DriverHostActor on every redundancy snapshot. + /// + /// The decision produced. + /// + void Publish(bool shouldDrainAlarmHistory); +} + +/// Thread-safe backed by a volatile field. +public sealed class RedundancyRoleView : IRedundancyRoleView +{ + // Seeded with the decision for "role unknown" rather than a bare false. An unpublished view and a + // node whose role nothing ever reports are genuinely the same situation, and they must behave the + // same: a deployment that runs no redundancy at all never publishes here, and defaulting closed + // would silently stop its alarm history forever. + private volatile bool _shouldDrainAlarmHistory = + PrimaryGatePolicy.ShouldDrainAlarmHistory(localRole: null, queueSharingPeerIsPrimary: false); + + /// + public bool ShouldDrainAlarmHistory => _shouldDrainAlarmHistory; + + /// + public void Publish(bool shouldDrainAlarmHistory) => _shouldDrainAlarmHistory = shouldDrainAlarmHistory; +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs index 0cddd870..9cdbeddb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs @@ -20,7 +20,9 @@ using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.Health; using ZB.MOM.WW.OtOpcUa.Runtime.Historian; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; +using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; +using ZB.MOM.WW.LocalDb; namespace ZB.MOM.WW.OtOpcUa.Runtime; @@ -39,7 +41,7 @@ public static class ServiceCollectionExtensions /// /// Registers shared runtime services. Currently binds /// to as the default; production deployments - /// override this with SqliteStoreAndForwardSink wrapping the HistorianGateway alarm writer. + /// override this with LocalDbStoreAndForwardSink wrapping the HistorianGateway alarm writer. /// Call this BEFORE AddAkka. /// /// The service collection to register with. @@ -65,12 +67,23 @@ public static class ServiceCollectionExtensions /// /// Config-gated durable alarm-historian sink. When the AlarmHistorian section has - /// Enabled=true, registers a (draining via the + /// Enabled=true, registers a (draining via the /// -supplied writer) as the , /// overriding the default. Otherwise a no-op (Null stays). /// The writer is injected so the durable downstream (the HistorianGateway alarm writer) can be /// supplied by the Host, which is the only project that references it. /// + /// + /// + /// The queue lives in the node's consolidated , which replicates it + /// to the redundant pair peer — so undelivered alarm history survives losing the node + /// holding it. That is also why the drain is gated on : + /// the Secondary holds a full replica of the Primary's queue, and an ungated drain there + /// would re-deliver every event. Both services are resolved from the provider, so a + /// deployment that enables this section without registering LocalDb fails loudly at + /// resolution rather than silently buffering to nowhere. + /// + /// /// The service collection to register with. /// The configuration carrying the AlarmHistorian section. /// @@ -87,21 +100,56 @@ public static class ServiceCollectionExtensions if (opts is not { Enabled: true }) return services; // leave the Null default from AddOtOpcUaRuntime foreach (var warning in opts.Validate()) - Serilog.Log.Logger.ForContext().Warning("Historian config: {HistorianConfigWarning}", warning); + Serilog.Log.Logger.ForContext().Warning("Historian config: {HistorianConfigWarning}", warning); + + // The view is a plain singleton so it exists even on nodes whose DriverHostActor never + // publishes to it — an unpublished view reads as "no peer, drain", which is the correct + // posture for a deployment that runs no redundancy at all. + services.TryAddSingleton(); + + // THE GATE ONLY APPLIES TO A REPLICATED QUEUE. Standing down is only ever safe because some + // other node holds the same rows and will send them instead — and the only node that does is + // this node's LocalDb replication peer. Without replication configured, these rows exist here + // and nowhere else, so deferring to anyone means they are never delivered by anyone. + // + // This is not hypothetical. The redundancy role is a CLUSTER-WIDE election + // (RedundancyStateActor keys on Akka's RoleLeader("driver")), while the queue is PAIR-LOCAL. + // On the docker-dev rig the elected driver Primary is a central node — which carries the + // driver Akka role, replicates nobody's LocalDb, and does not even run the alarm historian — + // so every site node dutifully suspended its drain in favour of a node that could not + // possibly deliver its events. Scoping the gate to "is my queue actually shared?" is what + // keeps the two scopes from disagreeing. + // BOTH halves of a pair share the queue, but only one of them dials: the initiator sets + // Replication:PeerAddress, its partner only sets SyncListenPort and waits. Testing the dial + // side alone would leave the listening half permanently ungated — one drainer per pair by + // accident rather than by role, and the wrong one whenever the roles swap. + var replicated = + !string.IsNullOrWhiteSpace(configuration["LocalDb:Replication:PeerAddress"]) + || !string.IsNullOrWhiteSpace(configuration["LocalDb:SyncListenPort"]); + + if (!replicated) + { + Serilog.Log.Logger.ForContext().Information( + "Alarm historian: LocalDb replication is not configured, so this node's queue is not " + + "shared with any peer and the Primary drain gate does not apply — this node always " + + "drains its own alarm queue."); + } services.AddSingleton(sp => { - // SqliteStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging). + // LocalDbStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging). // Resolve it off the host's configured static logger so the drain worker's WARN/INFO // lines land in the same sinks as the rest of the process. - var sink = new SqliteStoreAndForwardSink( - opts.DatabasePath, + var roleView = sp.GetRequiredService(); + var sink = new LocalDbStoreAndForwardSink( + sp.GetRequiredService(), writerFactory(opts, sp), - Serilog.Log.Logger.ForContext(), + Serilog.Log.Logger.ForContext(), batchSize: opts.BatchSize, capacity: opts.Capacity, deadLetterRetention: TimeSpan.FromDays(opts.DeadLetterRetentionDays), - maxAttempts: opts.MaxAttempts); + maxAttempts: opts.MaxAttempts, + drainGate: () => !replicated || roleView.ShouldDrainAlarmHistory); sink.StartDrainLoop(TimeSpan.FromSeconds(opts.DrainIntervalSeconds)); return sink; }); @@ -227,6 +275,21 @@ public static class ServiceCollectionExtensions // harnesses) rather than given a null-object, so DriverHostActor skips caching outright // instead of pretending to cache into a sink that drops everything. var deploymentArtifactCache = resolver.GetService(); + // Where this actor publishes its Primary-gate verdict for the alarm store-and-forward + // drain, which runs on a timer and so cannot read RedundancyStateChanged itself. + // Registered by AddAlarmHistorian; absent when no durable sink is configured, in which + // case there is nothing downstream to inform. + var redundancyRoleView = resolver.GetService(); + // Host of this node's LocalDb replication partner: the ONLY node that holds a copy of this + // node's alarm queue, and so the only node it may stand down in favour of. Null when this + // node dials nobody, which correctly means "never stand down". + var replicationPeerHost = + Uri.TryCreate( + resolver.GetService()?["LocalDb:Replication:PeerAddress"], + UriKind.Absolute, + out var peerUri) + ? peerUri.Host + : null; // Root script logger backs the ScriptedAlarm host's engine + script logging. Registered in // Host DI inside the hasDriver block; may be absent in some role configs / test harnesses, // in which case the DriverHostActor gracefully skips spawning the ScriptedAlarm host. @@ -345,7 +408,9 @@ public static class ServiceCollectionExtensions loggerFactory: loggerFactory, scriptRootLogger: scriptRootLogger, invokerFactory: invokerFactory, - deploymentArtifactCache: deploymentArtifactCache), + deploymentArtifactCache: deploymentArtifactCache, + redundancyRoleView: redundancyRoleView, + replicationPeerHost: replicationPeerHost), DriverHostActorName); registry.Register(driverHost); diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/SqliteStoreAndForwardSinkTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs similarity index 77% rename from tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/SqliteStoreAndForwardSinkTests.cs rename to tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs index 3f86e70f..da64ddf4 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/SqliteStoreAndForwardSinkTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs @@ -1,9 +1,12 @@ using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Serilog; using Serilog.Core; using Serilog.Events; using Shouldly; using Xunit; +using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests; @@ -14,22 +17,66 @@ namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests; /// capacity eviction, and retention-based dead-letter purge. /// [Trait("Category", "Unit")] -public sealed class SqliteStoreAndForwardSinkTests : IDisposable +public sealed class LocalDbStoreAndForwardSinkTests : IDisposable { private readonly string _dbPath; private readonly ILogger _log; + private ServiceProvider? _provider; + private ILocalDb? _db; /// Initializes a new test instance. - public SqliteStoreAndForwardSinkTests() + public LocalDbStoreAndForwardSinkTests() { _dbPath = Path.Combine(Path.GetTempPath(), $"otopcua-historian-{Guid.NewGuid():N}.sqlite"); _log = new LoggerConfiguration().MinimumLevel.Verbose().CreateLogger(); } + /// + /// A real temp-file local database with the buffer table created and registered, built on + /// first use. + /// + /// + /// A real database rather than a stub. The library has no in-memory mode, and a bare + /// would lack the zb_hlc_next() UDF the capture + /// triggers call — so every write to a registered table would fail closed. Registering here + /// also means these tests exercise the same capture path production replicates through. + /// + private ILocalDb Db + { + get + { + if (_db is not null) return _db; + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["LocalDb:Path"] = _dbPath }) + .Build(); + + _provider = new ServiceCollection() + .AddZbLocalDb(configuration, db => + { + using var connection = db.CreateConnection(); + AlarmSfSchema.Apply(connection); + db.RegisterReplicated(AlarmSfSchema.EventsTable); + }) + .BuildServiceProvider(); + + return _db = _provider.GetRequiredService(); + } + } + /// Cleans up test resources. public void Dispose() { - try { if (File.Exists(_dbPath)) File.Delete(_dbPath); } catch { } + _provider?.Dispose(); + + // Pooled connections keep a handle open past dispose; clearing the pools first is what + // makes the delete actually succeed. + SqliteConnection.ClearAllPools(); + + foreach (var path in new[] { _dbPath, $"{_dbPath}-wal", $"{_dbPath}-shm" }) + { + try { if (File.Exists(path)) File.Delete(path); } catch { /* best effort */ } + } } private sealed class FakeWriter : IAlarmHistorianWriter @@ -81,7 +128,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task EnqueueThenDrain_Ack_removes_row() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); sink.GetStatus().QueueDepth.ShouldBe(1); @@ -102,7 +149,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Drain_with_empty_queue_is_noop() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.DrainOnceAsync(CancellationToken.None); @@ -116,7 +163,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable { var writer = new FakeWriter(); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.RetryPlease); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); var before = sink.CurrentBackoff; @@ -133,7 +180,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable { var writer = new FakeWriter(); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.RetryPlease); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); await sink.DrainOnceAsync(CancellationToken.None); @@ -153,7 +200,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable var writer = new FakeWriter(); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.PermanentFail); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.Ack); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("bad"), CancellationToken.None); await sink.EnqueueAsync(Event("good"), CancellationToken.None); @@ -169,7 +216,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Writer_exception_treated_as_retry_for_whole_batch() { var writer = new FakeWriter { ThrowOnce = new InvalidOperationException("pipe broken") }; - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); await sink.DrainOnceAsync(CancellationToken.None); @@ -189,8 +236,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Capacity_eviction_drops_oldest_nondeadlettered_row() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, batchSize: 100, capacity: 3); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, batchSize: 100, capacity: 3); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); await sink.EnqueueAsync(Event("A2"), CancellationToken.None); @@ -217,8 +264,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable var writer = new FakeWriter(); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.PermanentFail); - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, deadLetterRetention: TimeSpan.FromDays(30), + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, deadLetterRetention: TimeSpan.FromDays(30), clock: () => clock); await sink.EnqueueAsync(Event("bad"), CancellationToken.None); @@ -238,7 +285,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable { var writer = new FakeWriter(); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.PermanentFail); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("bad"), CancellationToken.None); await sink.DrainOnceAsync(CancellationToken.None); @@ -257,7 +304,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Backoff_ladder_caps_at_60s() { var writer = new FakeWriter { DefaultOutcome = HistorianWriteOutcome.RetryPlease }; - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); @@ -278,8 +325,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task RetryPlease_dead_letters_row_after_MaxAttempts() { var writer = new FakeWriter { DefaultOutcome = HistorianWriteOutcome.RetryPlease }; - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, maxAttempts: 3); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, maxAttempts: 3); await sink.EnqueueAsync(Event("poison"), CancellationToken.None); @@ -316,17 +363,89 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable await NullAlarmHistorianSink.Instance.EnqueueAsync(Event("A1"), CancellationToken.None); } + /// + /// A closed drain gate must leave the queue completely alone — no writer call, no attempt + /// bump, no purge. + /// + /// + /// This is the property that keeps alarm delivery exactly-once across a redundant pair. + /// The buffer replicates, so the Secondary holds a full copy of the Primary's queue; a + /// Secondary that drained would re-send every event, continuously. + /// + [Fact] + public async Task Drain_does_nothing_while_the_gate_is_closed() + { + var writer = new FakeWriter(); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log, drainGate: () => false); + + await sink.EnqueueAsync(Event("A1"), CancellationToken.None); + await sink.EnqueueAsync(Event("A2"), CancellationToken.None); + + await sink.DrainOnceAsync(CancellationToken.None); + + writer.Batches.Count.ShouldBe(0, "a gated-off node must not deliver"); + + var status = sink.GetStatus(); + status.QueueDepth.ShouldBe(2, "the rows stay queued for whichever node holds the Primary role"); + status.DrainState.ShouldBe( + HistorianDrainState.NotPrimary, + "a Secondary's rising queue must be distinguishable from a stalled drain"); + } + + /// + /// The positive control for the case above: the same queue, the same sink, drains as soon + /// as the gate opens. Without this, a sink that was simply broken would pass the + /// gated-off assertion. + /// + [Fact] + public async Task Drain_delivers_the_retained_queue_once_the_gate_opens() + { + var writer = new FakeWriter(); + var isPrimary = false; + // ReSharper disable once AccessToModifiedClosure — flipping it mid-test is the point. + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log, drainGate: () => isPrimary); + + await sink.EnqueueAsync(Event("A1"), CancellationToken.None); + await sink.DrainOnceAsync(CancellationToken.None); + writer.Batches.Count.ShouldBe(0); + + isPrimary = true; // this node is promoted + await sink.DrainOnceAsync(CancellationToken.None); + + writer.Batches.Count.ShouldBe(1); + writer.Batches[0].Select(e => e.AlarmId).ShouldBe(["A1"]); + sink.GetStatus().QueueDepth.ShouldBe(0); + } + + /// + /// A gate that throws must be read as "not now", never as permission. Draining on a gate + /// fault would put a pair into dual-delivery precisely when its redundancy signal is sick. + /// + [Fact] + public async Task Drain_treats_a_throwing_gate_as_closed() + { + var writer = new FakeWriter(); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, drainGate: () => throw new InvalidOperationException("role unavailable")); + + await sink.EnqueueAsync(Event("A1"), CancellationToken.None); + await sink.DrainOnceAsync(CancellationToken.None); + + writer.Batches.Count.ShouldBe(0, "a faulted gate must fail closed"); + sink.GetStatus().QueueDepth.ShouldBe(1); + } + /// Verifies that the constructor rejects invalid arguments. [Fact] public void Ctor_rejects_bad_args() { var w = new FakeWriter(); - Should.Throw(() => new SqliteStoreAndForwardSink("", w, _log)); - Should.Throw(() => new SqliteStoreAndForwardSink(_dbPath, null!, _log)); - Should.Throw(() => new SqliteStoreAndForwardSink(_dbPath, w, null!)); - Should.Throw(() => new SqliteStoreAndForwardSink(_dbPath, w, _log, batchSize: 0)); - Should.Throw(() => new SqliteStoreAndForwardSink(_dbPath, w, _log, capacity: 0)); - Should.Throw(() => new SqliteStoreAndForwardSink(_dbPath, w, _log, maxAttempts: 0)); + Should.Throw(() => new LocalDbStoreAndForwardSink(null!, w, _log)); + Should.Throw(() => new LocalDbStoreAndForwardSink(Db, null!, _log)); + Should.Throw(() => new LocalDbStoreAndForwardSink(Db, w, null!)); + Should.Throw(() => new LocalDbStoreAndForwardSink(Db, w, _log, batchSize: 0)); + Should.Throw(() => new LocalDbStoreAndForwardSink(Db, w, _log, capacity: 0)); + Should.Throw(() => new LocalDbStoreAndForwardSink(Db, w, _log, maxAttempts: 0)); } /// Verifies that a disposed sink rejects enqueue operations. @@ -334,7 +453,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Disposed_sink_rejects_enqueue() { var writer = new FakeWriter(); - var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); sink.Dispose(); await Should.ThrowAsync( @@ -352,11 +471,14 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Drain_with_corrupt_payload_row_deadletters_it_and_keeps_good_rows_aligned() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + var clock = new TickingClock(); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log, clock: clock.Next); - // Row 1: good. Row 2: corrupt JSON (inserted directly). Row 3: good. + // Row 1: good. Row 2: corrupt JSON (inserted directly). Row 3: good. The shared ticking + // clock is what pins the corrupt row BETWEEN the good ones — which is the whole point of + // this case, as distinct from the corrupt-at-the-head case below. await sink.EnqueueAsync(Event("good-1"), CancellationToken.None); - InsertCorruptRow("corrupt"); + InsertCorruptRow("corrupt", clock.Next()); await sink.EnqueueAsync(Event("good-2"), CancellationToken.None); await sink.DrainOnceAsync(CancellationToken.None); @@ -383,9 +505,10 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Drain_with_corrupt_head_row_does_not_stall_queue() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + var clock = new TickingClock(); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log, clock: clock.Next); - InsertCorruptRow("corrupt-head"); + InsertCorruptRow("corrupt-head", clock.Next()); await sink.EnqueueAsync(Event("good-1"), CancellationToken.None); await sink.DrainOnceAsync(CancellationToken.None); @@ -409,7 +532,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task StartDrainLoop_honors_backoff_and_slows_cadence_under_retry() { var writer = new FakeWriter { DefaultOutcome = HistorianWriteOutcome.RetryPlease }; - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); @@ -436,7 +559,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task StartDrainLoop_keeps_steady_cadence_when_writer_is_healthy() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); sink.StartDrainLoop(TimeSpan.FromMilliseconds(30)); @@ -465,7 +588,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable // exception... so to exercise the *callback* catch we instead make the // fault originate from the writer itself but assert the loop self-heals. var writer = new ThrowingThenHealingWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); sink.StartDrainLoop(TimeSpan.FromMilliseconds(30)); @@ -490,7 +613,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Concurrent_enqueue_and_drain_do_not_throw_sqlite_busy() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); var faults = new List(); var enqueuers = Enumerable.Range(0, 4).Select(t => Task.Run(async () => @@ -561,7 +684,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Writer_returning_wrong_cardinality_outcomes_sets_backing_off_and_keeps_rows() { var writer = new WrongCardinalityWriter(returnExtraOutcome: true); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); await sink.EnqueueAsync(Event("A2"), CancellationToken.None); @@ -591,8 +714,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Capacity_eviction_increments_evicted_count_on_status() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, batchSize: 100, capacity: 3); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, batchSize: 100, capacity: 3); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); await sink.EnqueueAsync(Event("A2"), CancellationToken.None); @@ -616,7 +739,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task GetStatus_snapshot_is_consistent_under_concurrent_drain() { var writer = new FakeWriter { DefaultOutcome = HistorianWriteOutcome.RetryPlease }; - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); // Fill the queue. for (var i = 0; i < 10; i++) @@ -690,8 +813,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task EnqueueAsync_does_not_count_all_rows_on_every_call_below_capacity() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, batchSize: 100, capacity: 10_000); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, batchSize: 100, capacity: 10_000); for (var i = 0; i < 200; i++) await sink.EnqueueAsync(Event($"E{i}"), CancellationToken.None); @@ -705,7 +828,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable /// /// Regression for Core.AlarmHistorian-008: across every queue mutation (enqueue, /// Ack drain, PermanentFail drain, capacity eviction, RetryDeadLettered) the - /// queue depth reported by must + /// queue depth reported by must /// stay aligned with a fresh COUNT(*) against the database. Catches drift /// bugs in the in-memory counter introduced by the perf optimisation. /// @@ -713,8 +836,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Enqueue_and_drain_keep_queue_depth_consistent_with_storage() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, batchSize: 5, capacity: 8); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, batchSize: 5, capacity: 8); // Burst-enqueue below capacity — the in-memory counter must stay aligned with the // SELECT COUNT(*) that GetStatus runs. @@ -760,7 +883,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Counter_remains_consistent_under_concurrent_enqueue_and_drain() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); var enqueuers = Enumerable.Range(0, 3).Select(t => Task.Run(async () => { @@ -790,7 +913,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable /// Regression for Core.AlarmHistorian-012: when WriteBatchAsync throws /// the drain exits via re-throw /// without resetting _drainState, leaving the status surface permanently - /// showing Draining. The next call to + /// showing Draining. The next call to /// must reflect Idle or BackingOff, not the stale Draining /// state that was set at the top of the drain tick. /// @@ -798,7 +921,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Drain_cancelled_mid_write_resets_drain_state_to_not_draining() { var writer = new CancellableWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); using var cts = new CancellationTokenSource(); @@ -840,49 +963,68 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable /// COUNT(*) read directly from storage — proves the in-memory counter has not /// drifted from the persisted truth. /// - private void AssertQueueDepthMatchesStorage(SqliteStoreAndForwardSink sink) + private void AssertQueueDepthMatchesStorage(LocalDbStoreAndForwardSink sink) { - using var conn = new SqliteConnection($"Data Source={_dbPath}"); - conn.Open(); + using var conn = Db.CreateConnection(); using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM Queue WHERE DeadLettered = 0"; + cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0"; var live = (long)(cmd.ExecuteScalar() ?? 0L); sink.GetStatus().QueueDepth.ShouldBe(live, "GetStatus must agree with a fresh COUNT(*)"); } /// - /// Regression for Core.AlarmHistorian-014: - /// and must throw - /// after + /// Regression for Core.AlarmHistorian-014: + /// and must throw + /// after /// is called, consistent with the guards already present on - /// and - /// . + /// and + /// . /// [Fact] public void Disposed_sink_throws_ObjectDisposedException_on_GetStatus_and_RetryDeadLettered() { var writer = new FakeWriter(); - var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); sink.Dispose(); Should.Throw(() => sink.GetStatus()); Should.Throw(() => sink.RetryDeadLettered()); } - /// Insert a queue row whose PayloadJson cannot deserialize into an AlarmHistorianEvent. - private void InsertCorruptRow(string alarmId) + /// + /// Insert a queue row whose payload cannot deserialize into an AlarmHistorianEvent, stamped + /// at an explicit enqueue time. + /// + /// + /// The timestamp is a parameter rather than DateTime.UtcNow because the drain reads + /// in enqueued_at_utc order, and the tests that use this helper care precisely about + /// where the corrupt row sits in that order. Three unsynchronised clock reads can land on + /// one tick, which would make the interleaving — and therefore the test — a coin flip. + /// + private void InsertCorruptRow(string alarmId, DateTime enqueuedAt) { - using var conn = new SqliteConnection($"Data Source={_dbPath}"); - conn.Open(); + using var conn = Db.CreateConnection(); using var cmd = conn.CreateCommand(); cmd.CommandText = """ - INSERT INTO Queue (AlarmId, EnqueuedUtc, PayloadJson, AttemptCount) - VALUES ($alarmId, $enqueued, $payload, 0); + INSERT INTO alarm_sf_events (id, alarm_id, enqueued_at_utc, payload_json, attempt_count) + VALUES ($id, $alarmId, $enqueued, $payload, 0); """; + cmd.Parameters.AddWithValue("$id", Guid.NewGuid().ToString("N")); cmd.Parameters.AddWithValue("$alarmId", alarmId); - cmd.Parameters.AddWithValue("$enqueued", DateTime.UtcNow.ToString("O")); + cmd.Parameters.AddWithValue("$enqueued", enqueuedAt.ToString("O")); // JSON literal "null" round-trips through Deserialize as a null reference. cmd.Parameters.AddWithValue("$payload", "null"); cmd.ExecuteNonQuery(); } + + /// + /// Monotonic test clock — every read advances one second, so enqueue order is total and + /// the drain's ORDER BY enqueued_at_utc is deterministic. + /// + private sealed class TickingClock + { + private DateTime _now = new(2026, 7, 21, 0, 0, 0, DateTimeKind.Utc); + + public DateTime Next() => _now = _now.AddSeconds(1); + } } diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests.csproj b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests.csproj index 0cb9db4c..3f567086 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests.csproj +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests.csproj @@ -17,6 +17,14 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayAlarmWriterFactoryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayAlarmWriterFactoryTests.cs index ddbfc80a..868ab7f0 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayAlarmWriterFactoryTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayAlarmWriterFactoryTests.cs @@ -7,7 +7,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests; /// /// Alarm-write cutover seam test (T13). The Host's AddAlarmHistorian wiring drains the durable -/// SqliteStoreAndForwardSink through this factory, so it must yield the gateway-backed writer — +/// LocalDbStoreAndForwardSink through this factory, so it must yield the gateway-backed writer — /// sourcing the single gateway's connection from (endpoint/key/TLS), /// not the legacy Wonderware-shaped AlarmHistorian host/port. Built offline: the underlying /// channel dials lazily, so both the factory and the writer ctor perform no network I/O (a bogus, diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/HistorianGatewayClientAdapterTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/HistorianGatewayClientAdapterTests.cs index 1a86a92e..55719278 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/HistorianGatewayClientAdapterTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/HistorianGatewayClientAdapterTests.cs @@ -22,6 +22,61 @@ public sealed class HistorianGatewayClientAdapterTests Assert.NotNull(adapter); } + /// + /// A plaintext h2c gateway (an http:// endpoint with UseTls=false) must + /// construct, using nothing but documented defaults. + /// + /// + /// + /// Found by the LocalDb Phase 2 live gate, on the third consecutive crash-loop of the rig. + /// docs/Historian.md and CLAUDE.md both document http:// as a supported + /// transport ("scheme selects transport: https = TLS, http = h2c"), but it was in fact + /// unreachable: this factory forwarded the TLS-only options unconditionally, and + /// defaults to + /// , so it always sent RequireCertificateValidation=true — + /// which the client rejects outright when UseTls=false + /// ("RequireCertificateValidation is a TLS-only option and requires UseTls=true"). + /// + /// + /// So every h2c deployment crashed at startup, and the only way to avoid it was to + /// set AllowUntrustedServerCertificate=true — asserting a certificate posture for a + /// connection that has no certificate at all. + /// + /// + [Fact] + public void Adapter_constructs_for_plaintext_h2c_endpoint() + { + var opts = new ServerHistorianOptions + { + Enabled = true, Endpoint = "http://localhost:5222", ApiKey = "histgw_x_y", UseTls = false, + }; + + using var adapter = HistorianGatewayClientAdapter.Create(opts, NullLoggerFactory.Instance); + + adapter.ShouldNotBeNull(); + } + + /// + /// A pinned CA path is likewise TLS-only, and must not leak into an h2c client — the same + /// rejection, reached by a different field. + /// + [Fact] + public void Adapter_ignores_tls_only_settings_on_a_plaintext_endpoint() + { + var opts = new ServerHistorianOptions + { + Enabled = true, + Endpoint = "http://localhost:5222", + ApiKey = "histgw_x_y", + UseTls = false, + CaCertificatePath = "/etc/ssl/certs/leftover-from-a-tls-config.pem", + }; + + using var adapter = HistorianGatewayClientAdapter.Create(opts, NullLoggerFactory.Instance); + + adapter.ShouldNotBeNull(); + } + /// /// archreview 06/S-11 — an enabled historian with an empty Endpoint must fail with a /// named, config-key-carrying (defense-in-depth for any diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfConvergenceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfConvergenceTests.cs new file mode 100644 index 00000000..2ebf485a --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfConvergenceTests.cs @@ -0,0 +1,197 @@ +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.LocalDb; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.LocalDb; + +/// +/// LocalDb Phase 2 — two driver nodes replicating the alarm store-and-forward buffer over a real +/// loopback h2c transport, through the real fail-closed interceptor. +/// +/// +/// +/// This is the test the phase exists to pass. Everything upstream — the schema, the +/// registration, the sink rewire, the drain gate — can be individually green while a +/// redundant pair still fails to share the undelivered alarm history that is the whole +/// point of moving the queue here. +/// +/// +/// Rows are written through the production rather +/// than by hand-rolled INSERTs. The id derivation, the column set and the delete-on-ack +/// semantics under test are then the ones production actually uses. +/// +/// +[Collection("LocalDbPairConvergence")] +public sealed class AlarmSfConvergenceTests +{ + private static readonly Serilog.ILogger Log = new LoggerConfiguration().CreateLogger(); + + /// A writer that acknowledges everything — the drain's happy path. + private sealed class AckWriter : IAlarmHistorianWriter + { + public Task> WriteBatchAsync( + IReadOnlyList batch, CancellationToken cancellationToken) => + Task.FromResult>( + batch.Select(_ => HistorianWriteOutcome.Ack).ToList()); + } + + /// A writer that never acknowledges — a historian outage, which is when the buffer matters. + private sealed class OutageWriter : IAlarmHistorianWriter + { + public Task> WriteBatchAsync( + IReadOnlyList batch, CancellationToken cancellationToken) => + Task.FromResult>( + batch.Select(_ => HistorianWriteOutcome.RetryPlease).ToList()); + } + + private static AlarmHistorianEvent Event(string alarmId, int second) => new( + AlarmId: alarmId, + EquipmentPath: "line-1/eq-1", + AlarmName: "temp-high", + AlarmTypeName: "LimitAlarm", + Severity: AlarmSeverity.High, + EventKind: "Activated", + Message: "temperature high", + User: "system", + Comment: null, + TimestampUtc: new DateTime(2026, 7, 21, 0, 0, second, DateTimeKind.Utc)); + + /// The set of row ids on a node, ordered — equal sets on both nodes means converged. + private static async Task IdsAsync(ILocalDb db) + { + var rows = await db.QueryAsync( + "SELECT id FROM alarm_sf_events ORDER BY id", + static r => r.GetString(0)); + return string.Join(",", rows); + } + + private static async Task CountAsync(ILocalDb db) + { + var rows = await db.QueryAsync( + "SELECT COUNT(*) FROM alarm_sf_events", static r => r.GetInt64(0)); + return rows[0]; + } + + [Fact] + public async Task AlarmBurstOnA_ConvergesToB_AndOplogDrains() + { + await using var pair = new LocalDbPairHarness(); + await pair.StartAsync(); + + // The Primary's sink. Its drain is left unstarted so the burst stays queued and observable; + // an outage is exactly the state in which this buffer has to survive a node loss. + using var sink = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log); + for (var i = 0; i < 25; i++) + await sink.EnqueueAsync(Event($"eq/alarm-{i}", i), TestContext.Current.CancellationToken); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.B) == 25, + "node B to hold all 25 buffered alarm events"); + + (await IdsAsync(pair.B)).ShouldBe(await IdsAsync(pair.A)); + + // Oplog drains to empty once B acks: the steady state of a converged pair, and the proof + // the rows were genuinely acknowledged rather than merely sent. + await LocalDbPairHarness.WaitUntilAsync( + async () => await LocalDbPairHarness.OplogDepthAsync(pair.A) == 0, + "node A's oplog to drain after node B acknowledges the burst"); + } + + [Fact] + public async Task DeliveredRowsAreRemovedFromBothNodes() + { + // The no-redeliver-after-failover property, asserted at the data layer. An acknowledged row + // is DELETEd, and the delete has to reach the peer as a tombstone — otherwise a promoted + // standby would find the Primary's already-delivered events still queued and send them again. + await using var pair = new LocalDbPairHarness(); + await pair.StartAsync(); + + using var buffering = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log); + for (var i = 0; i < 5; i++) + await buffering.EnqueueAsync(Event($"eq/alarm-{i}", i), TestContext.Current.CancellationToken); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.B) == 5, + "node B to receive the buffered events"); + + // The historian comes back and the Primary drains. + using var draining = new LocalDbStoreAndForwardSink(pair.A, new AckWriter(), Log); + await draining.DrainOnceAsync(TestContext.Current.CancellationToken); + (await CountAsync(pair.A)).ShouldBe(0); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.B) == 0, + "node B to drop its copies once node A delivered them"); + + (await LocalDbPairHarness.TombstoneCountAsync(pair.B, "alarm_sf_events")) + .ShouldBe(5, "the deletes must arrive as tombstones, not vanish silently"); + } + + [Fact] + public async Task WritesWhileTransportDown_SurviveRejoin() + { + // The realistic outage shape: alarms keep arriving on the Primary while the pair link is + // down. Nothing may be lost when it comes back. + await using var pair = new LocalDbPairHarness(); + await pair.StartAsync(); + + using var sink = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log); + await sink.EnqueueAsync(Event("eq/before", 1), TestContext.Current.CancellationToken); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.B) == 1, + "the pre-outage event to reach node B"); + + await pair.StopPassiveAsync(); + + for (var i = 0; i < 10; i++) + await sink.EnqueueAsync(Event($"eq/during-{i}", 10 + i), TestContext.Current.CancellationToken); + + await pair.RestartPairAsync(); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.B) == 11, + "every event buffered during the outage to reach node B after the rejoin"); + + (await IdsAsync(pair.B)).ShouldBe(await IdsAsync(pair.A)); + } + + [Fact] + public async Task TheSameEventOnBothNodes_ConvergesToOneRow() + { + // Both adapters default-write while their redundancy role is unknown, so in the boot window + // the pair genuinely double-accepts the same fanned transition. Payload-derived ids are what + // turn that into one row instead of two; random GUIDs would leave the duplicate in the + // buffer for the eventual Primary to deliver twice. + await using var pair = new LocalDbPairHarness(); + await pair.StartAsync(); + + var shared = Event("eq/shared", 1); + using var sinkA = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log); + using var sinkB = new LocalDbStoreAndForwardSink(pair.B, new OutageWriter(), Log); + + await sinkA.EnqueueAsync(shared, TestContext.Current.CancellationToken); + await sinkB.EnqueueAsync(shared, TestContext.Current.CancellationToken); + + // A second, DISTINCT event on B. Without it the scenario would pass with replication + // switched off entirely — each node would hold its own single copy and "count == 1 on + // both" would be satisfied by two unconverged databases. + await sinkB.EnqueueAsync(Event("eq/only-on-b", 2), TestContext.Current.CancellationToken); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.A) == 2 && await CountAsync(pair.B) == 2, + "both nodes to hold two rows: the doubly-accepted event collapsed, plus B's own"); + + (await IdsAsync(pair.A)).ShouldBe(await IdsAsync(pair.B)); + + // Held, not merely reached: a late-arriving replica of the peer's copy of the shared event + // must not add a third row a moment later. + (await LocalDbPairHarness.HeldWithinAsync( + async () => await CountAsync(pair.A) > 2 || await CountAsync(pair.B) > 2, + TimeSpan.FromSeconds(3))) + .ShouldBeFalse("the duplicate must stay collapsed"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfLegacyMigratorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfLegacyMigratorTests.cs new file mode 100644 index 00000000..0a581633 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfLegacyMigratorTests.cs @@ -0,0 +1,333 @@ +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; +using Xunit; +using ZB.MOM.WW.LocalDb; +using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.LocalDb; + +/// +/// The one-time copy of the pre-consolidation alarm-historian.db queue into the +/// consolidated LocalDb. +/// +/// +/// +/// What is being protected is undelivered alarm history: rows that are in the legacy file +/// precisely because the historian could not be reached. Dropping them on upgrade would +/// discard exactly the audit trail the store-and-forward queue exists to preserve. +/// +/// +/// Run against a real temp-file built through the production +/// , so the registration order the migrator depends on is +/// the one the host actually runs — a hand-written schema here would prove only that the +/// test agrees with itself. +/// +/// +public sealed class AlarmSfLegacyMigratorTests : IDisposable +{ + private readonly string _dbPath = + Path.Combine(Path.GetTempPath(), $"otopcua-alarmsf-mig-{Guid.NewGuid():N}.db"); + + private readonly string _legacyPath = + Path.Combine(Path.GetTempPath(), $"otopcua-alarmsf-legacy-{Guid.NewGuid():N}.db"); + + private ServiceProvider? _provider; + + // ---- the cases --------------------------------------------------------------------------- + + [Fact] + public async Task Every_legacy_row_lands_in_the_consolidated_table() + { + SeedLegacy( + (1, "eq/a", Payload("eq/a"), 0, dead: 0), + (2, "eq/b", Payload("eq/b"), 3, dead: 0), + (3, "eq/c", Payload("eq/c"), 9, dead: 1)); + + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + (await CountAsync(db)).ShouldBe(3); + // Attempt counts and the dead-letter flag must survive: a migrated row that lost its + // attempt count would restart its retry budget, and one that lost its dead-letter flag + // would be re-sent to a historian that already refused it permanently. + var rows = await db.QueryAsync( + "SELECT alarm_id, attempt_count, dead_lettered FROM alarm_sf_events ORDER BY alarm_id", + r => (AlarmId: r.GetString(0), Attempts: r.GetInt64(1), Dead: r.GetInt64(2)), + parameters: null, + TestContext.Current.CancellationToken); + + rows.ShouldBe([("eq/a", 0L, 0L), ("eq/b", 3L, 0L), ("eq/c", 9L, 1L)]); + } + + [Fact] + public async Task Migrated_rows_enter_the_oplog() + { + // THE ordering pin. Capture is trigger-based, so rows copied in before + // RegisterReplicated would never enter the oplog and would never reach the peer — + // silently, and permanently. A migrator invoked too early passes every other case here. + SeedLegacy((1, "eq/a", Payload("eq/a"), 0, dead: 0)); + + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + var oplog = await db.QueryAsync( + "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'alarm_sf_events'", + r => r.GetInt64(0), + parameters: null, + TestContext.Current.CancellationToken); + + oplog[0].ShouldBe(1); + } + + [Fact] + public async Task A_second_run_is_a_no_op() + { + SeedLegacy((1, "eq/a", Payload("eq/a"), 0, dead: 0)); + var db = BuildDb(); + var config = ConfigWithLegacyPath(); + + AlarmSfLegacyMigrator.Migrate(db, config); + AlarmSfLegacyMigrator.Migrate(db, config); + + (await CountAsync(db)).ShouldBe(1); + File.Exists(_legacyPath).ShouldBeFalse("the legacy file is renamed once the copy commits"); + File.Exists(_legacyPath + ".migrated").ShouldBeTrue(); + } + + [Fact] + public async Task Re_migrating_the_same_rows_cannot_duplicate_them() + { + // The sidecar rename is the normal guard, but it is not the only one that has to hold: + // a crash between commit and rename leaves the file in place, and the next boot copies it + // again. Ids are derived from the payload, so the re-copy collides with itself and is + // ignored rather than doubling the queue. + SeedLegacy((1, "eq/a", Payload("eq/a"), 0, dead: 0)); + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + File.Move(_legacyPath + ".migrated", _legacyPath); // simulate the crash-before-rename + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + (await CountAsync(db)).ShouldBe(1); + } + + [Fact] + public async Task An_older_legacy_file_missing_a_column_still_migrates() + { + // A file written by an older build predates LastError. Naming a missing column in the + // SELECT throws "no such column", which would discard every row in the table rather than + // the one field. Intersecting the column list first means the data migrates and the + // absent column keeps its schema default. + SeedLegacyWithoutLastError((1, "eq/a", Payload("eq/a"))); + + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + (await CountAsync(db)).ShouldBe(1); + } + + [Fact] + public async Task A_failed_copy_leaves_the_legacy_file_untouched() + { + // The legacy table exists but its payload column is missing, so the required-column guard + // rejects the file. Nothing is copied and nothing is renamed: the operator still has the + // original to recover from, which is the whole point of renaming only after commit. + SeedUnrecognisedLegacy(); + + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + (await CountAsync(db)).ShouldBe(0); + File.Exists(_legacyPath).ShouldBeTrue("an un-copied file must not be renamed away"); + File.Exists(_legacyPath + ".migrated").ShouldBeFalse(); + } + + [Fact] + public async Task A_missing_legacy_file_is_a_clean_no_op() + { + // The expected case on a fresh install and on any node that never enabled the historian. + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + (await CountAsync(db)).ShouldBe(0); + } + + [Theory] + [InlineData(":memory:")] + [InlineData("file:thing?mode=memory")] + [InlineData("")] + public async Task Non_file_legacy_sources_are_no_ops(string configured) + { + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, Config(configured)); + + (await CountAsync(db)).ShouldBe(0); + } + + [Fact] + public async Task Two_nodes_migrating_the_same_event_converge_on_one_row() + { + // A warm pair's two legacy files overlap: HistorianAdapterActor default-writes while the + // redundancy role is unknown, so both nodes accepted the same transitions during every + // boot window. Node-prefixed ids would carry that duplication forward into the merged + // buffer permanently; payload-derived ids collapse it. + var shared = Payload("eq/shared"); + SeedLegacy((1, "eq/shared", shared, 0, dead: 0)); + + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + // The peer's file: a different legacy RowId for the very same event. + var peerLegacy = _legacyPath + ".peer"; + SeedLegacyAt(peerLegacy, (77, "eq/shared", shared, 0, dead: 0)); + AlarmSfLegacyMigrator.Migrate(db, Config(peerLegacy)); + + (await CountAsync(db)).ShouldBe(1); + + try { File.Delete(peerLegacy + ".migrated"); } catch { /* best effort */ } + } + + // ---- fixture ----------------------------------------------------------------------------- + + private ILocalDb BuildDb() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["LocalDb:Path"] = _dbPath }) + .Build(); + + _provider = new ServiceCollection() + .AddZbLocalDb(configuration, db => LocalDbSetup.OnReady(db, configuration)) + .BuildServiceProvider(); + + return _provider.GetRequiredService(); + } + + private IConfiguration ConfigWithLegacyPath() => Config(_legacyPath); + + private static IConfiguration Config(string legacyPath) => + new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["AlarmHistorian:DatabasePath"] = legacyPath, + }) + .Build(); + + private static async Task CountAsync(ILocalDb db) + { + var rows = await db.QueryAsync( + "SELECT COUNT(*) FROM alarm_sf_events", + r => r.GetInt64(0), + parameters: null, + TestContext.Current.CancellationToken); + return rows[0]; + } + + /// A payload shaped like a serialized AlarmHistorianEvent. + private static string Payload(string alarmId) => + $$"""{"AlarmId":"{{alarmId}}","EquipmentPath":"line/eq","AlarmName":"hi","TimestampUtc":"2026-07-20T00:00:00Z"}"""; + + private void SeedLegacy(params (long RowId, string AlarmId, string Payload, long Attempts, long dead)[] rows) => + SeedLegacyAt(_legacyPath, rows); + + /// Creates a legacy queue file with the full pre-cutover schema and the given rows. + private static void SeedLegacyAt( + string path, params (long RowId, string AlarmId, string Payload, long Attempts, long dead)[] rows) + { + using var conn = Open(path); + Exec(conn, """ + CREATE TABLE Queue ( + RowId INTEGER PRIMARY KEY AUTOINCREMENT, + AlarmId TEXT NOT NULL, + EnqueuedUtc TEXT NOT NULL, + PayloadJson TEXT NOT NULL, + AttemptCount INTEGER NOT NULL DEFAULT 0, + LastAttemptUtc TEXT NULL, + LastError TEXT NULL, + DeadLettered INTEGER NOT NULL DEFAULT 0 + ); + """); + + foreach (var row in rows) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + INSERT INTO Queue (RowId, AlarmId, EnqueuedUtc, PayloadJson, AttemptCount, DeadLettered) + VALUES ($id, $alarm, '2026-07-20T00:00:00.0000000Z', $payload, $attempts, $dead) + """; + cmd.Parameters.AddWithValue("$id", row.RowId); + cmd.Parameters.AddWithValue("$alarm", row.AlarmId); + cmd.Parameters.AddWithValue("$payload", row.Payload); + cmd.Parameters.AddWithValue("$attempts", row.Attempts); + cmd.Parameters.AddWithValue("$dead", row.dead); + cmd.ExecuteNonQuery(); + } + } + + /// A legacy file from an older build, predating the LastError column. + private void SeedLegacyWithoutLastError(params (long RowId, string AlarmId, string Payload)[] rows) + { + using var conn = Open(_legacyPath); + Exec(conn, """ + CREATE TABLE Queue ( + RowId INTEGER PRIMARY KEY AUTOINCREMENT, + AlarmId TEXT NOT NULL, + EnqueuedUtc TEXT NOT NULL, + PayloadJson TEXT NOT NULL, + AttemptCount INTEGER NOT NULL DEFAULT 0, + DeadLettered INTEGER NOT NULL DEFAULT 0 + ); + """); + + foreach (var row in rows) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + INSERT INTO Queue (RowId, AlarmId, EnqueuedUtc, PayloadJson) + VALUES ($id, $alarm, '2026-07-20T00:00:00.0000000Z', $payload) + """; + cmd.Parameters.AddWithValue("$id", row.RowId); + cmd.Parameters.AddWithValue("$alarm", row.AlarmId); + cmd.Parameters.AddWithValue("$payload", row.Payload); + cmd.ExecuteNonQuery(); + } + } + + /// A file with a Queue table this build cannot read — no payload column. + private void SeedUnrecognisedLegacy() + { + using var conn = Open(_legacyPath); + Exec(conn, "CREATE TABLE Queue (RowId INTEGER PRIMARY KEY AUTOINCREMENT, Something TEXT);"); + Exec(conn, "INSERT INTO Queue (Something) VALUES ('x');"); + } + + private static SqliteConnection Open(string path) + { + var conn = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = path }.ToString()); + conn.Open(); + return conn; + } + + private static void Exec(SqliteConnection conn, string sql) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + cmd.ExecuteNonQuery(); + } + + public void Dispose() + { + _provider?.Dispose(); + SqliteConnection.ClearAllPools(); + + foreach (var baseName in new[] { _dbPath, _legacyPath, _legacyPath + ".migrated" }) + { + foreach (var suffix in new[] { "", "-wal", "-shm" }) + { + try { File.Delete(baseName + suffix); } catch { /* best effort */ } + } + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/LocalDbPairHarness.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/LocalDbPairHarness.cs index a221dee1..abfd7e75 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/LocalDbPairHarness.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/LocalDbPairHarness.cs @@ -265,7 +265,7 @@ public sealed class LocalDbPairHarness : IAsyncDisposable .Build(); return new ServiceCollection() - .AddZbLocalDb(config, LocalDbSetup.OnReady) + .AddZbLocalDb(config, db => LocalDbSetup.OnReady(db, config)) .BuildServiceProvider(); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs index 2db014c2..3e45a17d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs @@ -39,14 +39,14 @@ public sealed class LocalDbSetupTests : IDisposable .Build(); _provider = new ServiceCollection() - .AddZbLocalDb(configuration, LocalDbSetup.OnReady) + .AddZbLocalDb(configuration, db => LocalDbSetup.OnReady(db, configuration)) .BuildServiceProvider(); return _provider.GetRequiredService(); } [Fact] - public void OnReady_RegistersExactlyTheTwoDeploymentTables() + public void OnReady_RegistersExactlyTheThreeReplicatedTables() { // Both directions are load-bearing. "No fewer" catches a dropped RegisterReplicated call // (that table then never replicates). "No more" catches an accidental registration — @@ -54,7 +54,7 @@ public sealed class LocalDbSetupTests : IDisposable var db = BuildDb(); db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal) - .ShouldBe(["deployment_artifacts", "deployment_pointer"]); + .ShouldBe(["alarm_sf_events", "deployment_artifacts", "deployment_pointer"]); } [Fact] @@ -78,6 +78,50 @@ public sealed class LocalDbSetupTests : IDisposable db.ReplicatedTables["deployment_pointer"].PkColumns.ShouldBe(["cluster_id"]); } + [Fact] + public void AlarmSfEvents_PkIsTheAppMintedId() + { + // A single TEXT PK the application mints. The legacy queue this table replaces keyed on + // `RowId INTEGER PRIMARY KEY AUTOINCREMENT`, which LWW cannot replicate: two nodes would + // independently allocate rowid 7 for different alarms and silently overwrite each other. + var db = BuildDb(); + + db.ReplicatedTables["alarm_sf_events"].PkColumns.ShouldBe(["id"]); + } + + [Fact] + public async Task AlarmSfEventRows_EnterTheOplog() + { + // Same ordering assertion as the deployment-pointer test below, for the table Phase 2 adds. + // Worth its own case because the alarm table is registered by a different call site, and a + // registration added ahead of its DDL would fail loudly while one added after the migrator + // would fail silently. + var db = BuildDb(); + + await db.ExecuteAsync( + """ + INSERT INTO alarm_sf_events + (id, alarm_id, enqueued_at_utc, payload_json, attempt_count) + VALUES (@Id, @AlarmId, @EnqueuedAtUtc, @PayloadJson, 0) + """, + new + { + Id = new string('c', 64), + AlarmId = "equip/alarm-1", + EnqueuedAtUtc = "2026-07-21T00:00:00.0000000Z", + PayloadJson = """{"AlarmId":"equip/alarm-1"}""", + }, + TestContext.Current.CancellationToken); + + var oplogRows = await db.QueryAsync( + "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'alarm_sf_events'", + r => r.GetInt32(0), + parameters: null, + TestContext.Current.CancellationToken); + + oplogRows[0].ShouldBe(1); + } + [Fact] public async Task RowsWrittenAfterOnReady_EnterTheOplog() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbWiringTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbWiringTests.cs index 77060548..82f6c36e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbWiringTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbWiringTests.cs @@ -84,7 +84,7 @@ public sealed class LocalDbWiringTests : IDisposable } [Fact] - public void DriverGraph_LocalDbResolvesAsASingletonWithBothCacheTablesRegistered() + public void DriverGraph_LocalDbResolvesAsASingletonWithEveryReplicatedTableRegistered() { var sp = BuildDriverGraph(); @@ -95,7 +95,7 @@ public sealed class LocalDbWiringTests : IDisposable // Exact set, both directions — a dropped registration replicates nothing; an extra one costs // three triggers and oplog volume on every write. first.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal) - .ShouldBe(["deployment_artifacts", "deployment_pointer"]); + .ShouldBe(["alarm_sf_events", "deployment_artifacts", "deployment_pointer"]); } [Fact] diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ServerHistorianOptionsValidatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ServerHistorianOptionsValidatorTests.cs index d374c6b9..54baf356 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ServerHistorianOptionsValidatorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ServerHistorianOptionsValidatorTests.cs @@ -62,11 +62,18 @@ public sealed class ServerHistorianOptionsValidatorTests result.Failures.ShouldContain(f => f.Contains("ServerHistorian:Endpoint")); } - /// Enabled with a valid absolute https endpoint succeeds. + /// + /// Enabled with a valid absolute https endpoint and a key succeeds. The key is not + /// incidental: this case previously omitted it and still asserted success, which pinned a config + /// that actually crash-loops the host — see . + /// [Fact] public void Enabled_with_valid_https_endpoint_succeeds() { - var result = Sut().Validate(null, new ServerHistorianOptions { Enabled = true, Endpoint = "https://host:5222" }); + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://host:5222", ApiKey = "histgw_id_secret", + }); result.Succeeded.ShouldBeTrue(); } @@ -96,16 +103,145 @@ public sealed class ServerHistorianOptionsValidatorTests f.Contains("ServerHistorian:Endpoint") && f.Contains("AlarmHistorian:Enabled")); } - /// Alarm-only mode with a valid endpoint succeeds. + /// Alarm-only mode with a valid endpoint and a key succeeds. [Fact] public void Disabled_but_alarm_historian_enabled_with_valid_endpoint_succeeds() { var result = Sut(alarmHistorianEnabled: true) - .Validate(null, new ServerHistorianOptions { Enabled = false, Endpoint = "https://host:5222" }); + .Validate(null, new ServerHistorianOptions + { + Enabled = false, Endpoint = "https://host:5222", ApiKey = "histgw_id_secret", + }); result.Succeeded.ShouldBeTrue(); } + /// + /// An empty ApiKey on a consumed section fails, because it crashes — it does not + /// degrade. + /// + /// + /// + /// Found by the LocalDb Phase 2 live gate. The rig booted with + /// AlarmHistorian:Enabled=true and a valid endpoint but no key, and every driver + /// node crash-looped on + /// ArgumentException: The gateway API key must not be empty (Parameter 'ApiKey') + /// thrown by HistorianGatewayClientOptions.Validate() — reached through + /// GatewayHistorian.CreateAlarmWriter during Akka startup. + /// + /// + /// That is the *same* factory path, and the same crash-loop-under-a-restart-policy + /// failure mode, this validator was built to convert into a named + /// OptionsValidationException. The previous "empty ApiKey degrades, the gateway + /// just rejects calls" premise was simply wrong: the client validates its own options + /// at construction, so the process never reaches the point of making a call. + /// + /// + [Fact] + public void Consumed_with_empty_api_key_fails() + { + var result = Sut(alarmHistorianEnabled: true) + .Validate(null, new ServerHistorianOptions + { + Enabled = false, Endpoint = "https://host:5222", ApiKey = "", + }); + + result.Failed.ShouldBeTrue(); + result.Failures.ShouldContain(f => + f.Contains("ServerHistorian:ApiKey") && f.Contains("AlarmHistorian:Enabled")); + } + + /// The read path has the identical crash, so it is gated identically. + [Fact] + public void Enabled_with_empty_api_key_fails() + { + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://host:5222", ApiKey = "", + }); + + result.Failed.ShouldBeTrue(); + result.Failures.ShouldContain(f => f.Contains("ServerHistorian:ApiKey")); + } + + /// + /// The key must never be echoed. An endpoint is not a secret and is quoted to make the error + /// actionable; a key is, so the failure names only the setting. + /// + [Fact] + public void Api_key_failure_never_echoes_the_key() + { + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://host:5222", ApiKey = " ", + }); + + result.Failed.ShouldBeTrue(); + result.Failures.ShouldNotContain(f => f.Contains(" ", StringComparison.Ordinal)); + } + + /// A section nobody consumes may be keyless — no failure. + [Fact] + public void Disabled_with_empty_api_key_succeeds() + { + var result = Sut().Validate(null, new ServerHistorianOptions { Enabled = false, ApiKey = "" }); + + result.Succeeded.ShouldBeTrue(); + } + + /// + /// UseTls=true against an http:// endpoint fails — the third member of the + /// crash-at-construction family, and the easiest of the three for an operator to trip. + /// + /// + /// Also found by the Phase 2 live gate, immediately after the ApiKey fix: the rig points + /// at a deliberately unresolvable http:// endpoint, and UseTls defaults to + /// , so every node crash-looped a second time on + /// ArgumentException: UseTls requires an https gateway endpoint. Switching an endpoint + /// from https to http without clearing UseTls is an ordinary migration slip, and it takes + /// the host down rather than degrading it. + /// + [Fact] + public void Consumed_with_use_tls_against_http_endpoint_fails() + { + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "http://host:5222", ApiKey = "histgw_id_secret", UseTls = true, + }); + + result.Failed.ShouldBeTrue(); + result.Failures.ShouldContain(f => + f.Contains("ServerHistorian:UseTls") && f.Contains("http://host:5222")); + } + + /// An http endpoint with TLS explicitly off is the valid h2c shape — no failure. + [Fact] + public void Consumed_with_http_endpoint_and_tls_off_succeeds() + { + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "http://host:5222", ApiKey = "histgw_id_secret", UseTls = false, + }); + + result.Succeeded.ShouldBeTrue(); + } + + /// + /// The mirror slip — UseTls=false against an https:// endpoint — also fails, so the + /// scheme and the flag are pinned to agree in both directions rather than in one. + /// + [Fact] + public void Consumed_with_tls_off_against_https_endpoint_fails() + { + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://host:5222", ApiKey = "histgw_id_secret", UseTls = false, + }); + + result.Failed.ShouldBeTrue(); + result.Failures.ShouldContain(f => f.Contains("ServerHistorian:UseTls")); + } + /// /// Wiring guard (the "register-AND-consume" trap): resolving IOptions<ServerHistorianOptions>.Value /// after throws diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs new file mode 100644 index 00000000..c46952fa --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs @@ -0,0 +1,211 @@ +using Akka.Actor; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; +using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; + +/// +/// The bridge that carries the Primary-gate decision out of the actor and into +/// , where the alarm store-and-forward drain can read it. +/// +/// +/// +/// The drain runs on a timer owned by the sink, not on a mailbox, so it cannot receive +/// itself — but it must be Primary-scoped, because +/// the queue it drains replicates to the pair peer and two draining nodes would deliver +/// every alarm event twice. +/// +/// +/// These cases deliberately DIVERGE from in the +/// unknown-role case: the write gate denies when a peer may exist, this one allows. A duplicate +/// history row is cheap and already contemplated by the at-least-once contract; a silently +/// un-drained queue is not. See AlarmHistoryDrainGatePolicyTests. +/// +/// +public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase +{ + private static readonly NodeId TestNode = NodeId.Parse("driver-roleview-test"); + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5); + + /// + /// Before any snapshot arrives the view must read open, not closed. + /// + /// + /// A deployment that runs no redundancy never publishes here at all. Defaulting closed + /// would silently stop its alarm history forever — the exact failure the drain gate exists + /// to avoid causing. + /// + [Fact] + public void An_unpublished_view_reads_as_primary() + { + new RedundancyRoleView().ShouldDrainAlarmHistory.ShouldBeTrue(); + } + + /// A Secondary closes the view — but only because the snapshot also names a Primary. + [Fact] + public void Secondary_role_closes_the_view_when_a_primary_exists() + { + var view = SpawnAndTellRole(RedundancyRole.Secondary, driverMemberCount: 2); + + AwaitAssert(() => view.Published.ShouldBe([false]), duration: Timeout); + } + + /// + /// A Secondary whose Primary is some OTHER pair's node keeps draining: that node holds none of + /// this node's rows, so deferring to it would deliver them never. + /// + [Fact] + public void Secondary_role_keeps_the_view_open_when_the_primary_is_not_its_peer() + { + var view = new RecordingRoleView(); + SpawnHost(view, driverMemberCount: 4).Tell(new RedundancyStateChanged( + [ + new NodeRedundancyState(TestNode, RedundancyRole.Secondary, + IsClusterLeader: false, IsRoleLeaderForDriver: false, AsOfUtc: DateTime.UtcNow), + // A Primary DOES exist — in another pair. It is not this node's replication peer. + new NodeRedundancyState(NodeId.Parse("some-other-pair:4053"), RedundancyRole.Primary, + IsClusterLeader: true, IsRoleLeaderForDriver: true, AsOfUtc: DateTime.UtcNow), + ], + CorrelationId.NewId())); + + AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout); + } + + [Fact] + public void Primary_role_opens_the_view() + { + var view = SpawnAndTellRole(RedundancyRole.Primary, driverMemberCount: 2); + + // Published, not current — `true` is the seed. See the note in the unknown-role theory. + AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout); + } + + /// + /// Unknown role ⇒ the view stays open, even with driver peers present — the deliberate + /// divergence from the write gate's default-DENY. + /// + /// + /// + /// This case previously asserted the opposite, on the stated goal of mirroring the write + /// gate. The Phase 2 live gate showed that goal to be wrong for this consumer: the rig runs + /// four driver nodes in one cluster with no Redundancy section, so every role was + /// unknown and the driver count was 4 — and every node logged "Historian drain suspended", + /// including the two unpaired site-b nodes with no peer and no replication. Alarm history + /// drained nowhere, where before Phase 2 it drained fine. + /// + /// + /// See AlarmHistoryDrainGatePolicyTests for why the asymmetry of the two error costs + /// makes fail-open correct here and fail-closed correct for device writes. + /// + /// + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(4)] + public void Unknown_role_leaves_the_view_open_whatever_the_peer_count(int driverMemberCount) + { + // A snapshot that never names this node is how the role stays unknown in practice — the + // documented identity-mismatch shape, not a contrived case. + var view = SpawnAndTellForeignSnapshot(driverMemberCount); + + // Assert on what was PUBLISHED, never on the current value. `true` is also the seeded + // default, so `AwaitAssert(() => view.ShouldDrainAlarmHistory.ShouldBeTrue())` passes at the + // first poll — before the actor has processed the snapshot — and therefore passes just as + // happily when the publish is wrong or absent. Verified: with the old write-gate verdict + // restored, the current-value form stayed green and only this form goes red. + AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout); + } + + /// A demotion must move the view, not just an initial promotion. + [Fact] + public void A_demotion_closes_a_previously_open_view() + { + var view = new RecordingRoleView(); + var host = SpawnHost(view, driverMemberCount: 2); + + host.Tell(Snapshot(TestNode, RedundancyRole.Primary)); + AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout); + + host.Tell(Snapshot(TestNode, RedundancyRole.Secondary)); + AwaitAssert(() => view.Published.ShouldBe([true, false]), duration: Timeout); + } + + // ---------------- helpers ---------------- + + /// + /// A view that remembers every published decision, so a test can assert on the actor's + /// action rather than on a state the seed already satisfies. + /// + private sealed class RecordingRoleView : IRedundancyRoleView + { + private readonly List _published = []; + private readonly Lock _gate = new(); + private readonly RedundancyRoleView _inner = new(); + + /// Every value published so far, in order. + public IReadOnlyList Published + { + get { lock (_gate) return _published.ToArray(); } + } + + public bool ShouldDrainAlarmHistory => _inner.ShouldDrainAlarmHistory; + + public void Publish(bool shouldDrainAlarmHistory) + { + lock (_gate) _published.Add(shouldDrainAlarmHistory); + _inner.Publish(shouldDrainAlarmHistory); + } + } + + private RecordingRoleView SpawnAndTellRole(RedundancyRole role, int driverMemberCount) + { + var view = new RecordingRoleView(); + SpawnHost(view, driverMemberCount).Tell(Snapshot(TestNode, role)); + return view; + } + + private RecordingRoleView SpawnAndTellForeignSnapshot(int driverMemberCount) + { + var view = new RecordingRoleView(); + SpawnHost(view, driverMemberCount).Tell(Snapshot(NodeId.Parse("some-other-node"), RedundancyRole.Primary)); + return view; + } + + /// The peer this node replicates with — the only node it may ever stand down for. + private const string PeerHost = "the-peer"; + + private IActorRef SpawnHost(IRedundancyRoleView view, int driverMemberCount) => + Sys.ActorOf(DriverHostActor.Props( + NewInMemoryDbFactory(), + TestNode, + coordinator: null, + driverMemberCountProvider: () => driverMemberCount, + redundancyRoleView: view, + replicationPeerHost: PeerHost)); + + /// + /// A snapshot of a real pair: this node in , plus a peer holding the + /// other half. The peer matters — a Secondary only stands down when a Primary exists, so a + /// single-entry snapshot would silently change what these cases test. + /// + private static RedundancyStateChanged Snapshot(NodeId node, RedundancyRole role) => + new( + [ + new NodeRedundancyState(node, role, + IsClusterLeader: role == RedundancyRole.Primary, + IsRoleLeaderForDriver: role == RedundancyRole.Primary, + AsOfUtc: DateTime.UtcNow), + new NodeRedundancyState( + NodeId.Parse($"{PeerHost}:4053"), + role == RedundancyRole.Primary ? RedundancyRole.Secondary : RedundancyRole.Primary, + IsClusterLeader: role != RedundancyRole.Primary, + IsRoleLeaderForDriver: role != RedundancyRole.Primary, + AsOfUtc: DateTime.UtcNow), + ], + CorrelationId.NewId()); +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/PrimaryGatePolicyTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/PrimaryGatePolicyTests.cs index 07dac1d6..7a1262a9 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/PrimaryGatePolicyTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/PrimaryGatePolicyTests.cs @@ -36,3 +36,86 @@ public sealed class PrimaryGatePolicyTests public void Unknown_role_resolves_by_membership(int members, bool expected) => PrimaryGatePolicy.ShouldServiceAsPrimary(null, members).ShouldBe(expected); } + +/// +/// The alarm-history drain gate, which deliberately does not mirror the device-write gate. +/// +/// +/// +/// Why a second policy is correct here. The write gate protects a shared field device: +/// two nodes driving one PLC is dangerous and irreversible, so an unknown role with a peer +/// present must deny. The drain writes to an append-only historian instead, and Phase 2's +/// delivery contract is already at-least-once across a failover, by design — identical +/// events even collapse to one row, because ids are a hash of the payload. +/// +/// +/// So the two error costs are wildly asymmetric: a false allow costs a duplicate history row, +/// while a false deny silently stops the alarm audit trail and eventually evicts it at the +/// capacity wall. This gate therefore keys on the role alone and opens when the role is +/// unknown. +/// +/// +/// Found by the LocalDb Phase 2 live gate. The rig runs four driver nodes in one Akka +/// cluster with no Redundancy section, so every node's role was unknown and the +/// cluster-wide driver count was 4 — and every node, including the two unpaired site-b nodes +/// that have no peer and no replication at all, logged "Historian drain suspended". Nothing +/// drained anywhere. Before Phase 2 that deployment drained fine, because the drain was ungated. +/// +/// +public sealed class AlarmHistoryDrainGatePolicyTests +{ + [Theory] + // A Primary always drains its own queue. + [InlineData(RedundancyRole.Primary, true, true)] + [InlineData(RedundancyRole.Primary, false, true)] + // A Secondary stands down ONLY when a Primary actually exists to stand up. + [InlineData(RedundancyRole.Secondary, true, false)] + [InlineData(RedundancyRole.Detached, true, false)] + [InlineData(RedundancyRole.Secondary, false, true)] + [InlineData(RedundancyRole.Detached, false, true)] + public void A_known_role_stands_down_only_for_a_primary_that_exists( + RedundancyRole role, bool queueSharingPeerIsPrimary, bool expected) + => PrimaryGatePolicy.ShouldDrainAlarmHistory(role, queueSharingPeerIsPrimary).ShouldBe(expected); + + /// + /// Unknown role ⇒ drain, whoever else is out there. The first case the live gate broke on, and + /// the one that separates this policy from . + /// + [Theory] + [InlineData(true)] + [InlineData(false)] + public void An_unknown_role_drains(bool queueSharingPeerIsPrimary) + => PrimaryGatePolicy.ShouldDrainAlarmHistory(null, queueSharingPeerIsPrimary).ShouldBeTrue(); + + /// + /// A Secondary whose Primary is NOT its queue-sharing peer keeps draining — the deepest of the + /// live gate's findings. + /// + /// + /// RedundancyStateActor derives roles from Akka's cluster-wide RoleLeader("driver"), + /// but the alarm queue is pair-local. On the docker-dev rig the elected Primary is a CENTRAL node + /// — it carries the driver Akka role, replicates nobody's LocalDb, and does not even run the + /// alarm historian — so under a plain "a Primary exists ⇒ stand down" rule every site node + /// suspended its drain in favour of a node that could not possibly deliver its events. The + /// buffer then grows on every node until the capacity wall evicts the oldest: silent, permanent + /// loss of the audit trail the queue exists to protect. + /// + [Fact] + public void A_secondary_whose_peer_is_not_the_primary_keeps_draining() + { + // A Primary exists somewhere in the cluster, but it does not hold this node's rows. + PrimaryGatePolicy.ShouldDrainAlarmHistory(RedundancyRole.Secondary, queueSharingPeerIsPrimary: false) + .ShouldBeTrue("a Secondary must not defer to a Primary that cannot deliver its events"); + } + + /// + /// The divergence, pinned explicitly: with a peer present and no role known, the write gate + /// denies and the drain gate allows. If someone later "harmonises" the two, this fails. + /// + [Fact] + public void The_two_gates_deliberately_disagree_when_the_role_is_unknown_and_a_peer_exists() + { + PrimaryGatePolicy.ShouldServiceAsPrimary(localRole: null, driverMemberCount: 4).ShouldBeFalse(); + PrimaryGatePolicy.ShouldDrainAlarmHistory(localRole: null, queueSharingPeerIsPrimary: true).ShouldBeTrue(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs index 8100f19f..32531012 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs @@ -1,22 +1,25 @@ +using Microsoft.Data.Sqlite; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Shouldly; using Xunit; +using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; using ZB.MOM.WW.OtOpcUa.Runtime.Historian; +using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Historian; /// /// Verifies the config-gated AddAlarmHistorian registration: when the /// AlarmHistorian section is absent or disabled the -/// default survives; when it is enabled a real wins +/// default survives; when it is enabled a real wins /// (last-registration-wins over the TryAddSingleton Null default). /// public sealed class AlarmHistorianRegistrationTests { - /// A no-op writer the factory hands the Sqlite sink; never actually invoked in these tests. + /// A no-op writer the factory hands the sink; never actually invoked in these tests. private sealed class FakeWriter : IAlarmHistorianWriter { public Task> WriteBatchAsync( @@ -65,34 +68,164 @@ public sealed class AlarmHistorianRegistrationTests } [Fact] - public void Section_enabled_registers_sqlite_sink() + public void Section_enabled_registers_the_localdb_sink() { var tempDir = Path.Combine(Path.GetTempPath(), "otopcua-alarmhist-test-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(tempDir); - var dbPath = Path.Combine(tempDir, "alarm-historian-test.db"); + var dbPath = Path.Combine(tempDir, "node-local.db"); try { - var services = BaseServices(); var config = ConfigFrom(new Dictionary { ["AlarmHistorian:Enabled"] = "true", - ["AlarmHistorian:DatabasePath"] = dbPath, + ["LocalDb:Path"] = dbPath, }); + var services = BaseServices(); + services.AddZbLocalDb(config, db => + { + using var connection = db.CreateConnection(); + AlarmSfSchema.Apply(connection); + db.RegisterReplicated(AlarmSfSchema.EventsTable); + }); services.AddAlarmHistorian(config, (_, _) => new FakeWriter()); using (var provider = services.BuildServiceProvider()) { - provider.GetRequiredService().ShouldBeOfType(); + provider.GetRequiredService().ShouldBeOfType(); } // dispose stops the drain loop + releases the SQLite file handle } finally { + SqliteConnection.ClearAllPools(); try { Directory.Delete(tempDir, recursive: true); } catch (IOException) { /* best effort */ } } } + /// + /// A node whose LocalDb is not replicated drains its own queue regardless of redundancy role. + /// + /// + /// + /// Standing down is only safe because a peer holds the same rows and will send them + /// instead. Without LocalDb:Replication:PeerAddress there is no such peer: the rows + /// are here and nowhere else, so any deferral means nobody ever delivers them. + /// + /// + /// Found by the Phase 2 live gate. The redundancy role is a CLUSTER-WIDE election while the + /// queue is PAIR-LOCAL, and on the docker-dev rig the elected driver Primary turned out to + /// be a central node — one that carries the driver Akka role, replicates nobody's LocalDb + /// and does not even run the alarm historian. Every site node suspended its drain in favour + /// of a node that could not possibly deliver its events. + /// + /// + [Theory] + // Neither key: an unpaired node. Its rows exist nowhere else. + [InlineData(null, null, true)] + // The dialing half of a pair — gated, because its partner holds the same rows. + [InlineData("http://peer:9001", null, false)] + // The LISTENING half. Same shared queue, but it never dials, so a PeerAddress-only test would + // wrongly class it as unpaired and leave it permanently ungated. + [InlineData(null, "9001", false)] + public async Task Only_an_unreplicated_node_ignores_the_role_view( + string? peerAddress, string? syncListenPort, bool expectDrain) + { + var tempDir = Path.Combine(Path.GetTempPath(), "otopcua-alarmhist-test-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + var dbPath = Path.Combine(tempDir, "node-local.db"); + + try + { + var config = ConfigFrom(new Dictionary + { + ["AlarmHistorian:Enabled"] = "true", + ["LocalDb:Path"] = dbPath, + ["LocalDb:Replication:PeerAddress"] = peerAddress, + ["LocalDb:SyncListenPort"] = syncListenPort, + }); + + var services = BaseServices(); + services.AddZbLocalDb(config, db => + { + using var connection = db.CreateConnection(); + AlarmSfSchema.Apply(connection); + db.RegisterReplicated(AlarmSfSchema.EventsTable); + }); + + // A role view pinned SHUT: this node is a Secondary and a Primary exists elsewhere. + services.AddSingleton(new StandDownRoleView()); + services.AddAlarmHistorian(config, (_, _) => new FakeWriter()); + + using (var provider = services.BuildServiceProvider()) + { + var sink = provider.GetRequiredService(); + await sink.EnqueueAsync(SampleEvent, CancellationToken.None); + + await ((LocalDbStoreAndForwardSink)sink).DrainOnceAsync(CancellationToken.None); + + var status = sink.GetStatus(); + if (expectDrain) + { + // Delivered and removed — not parked behind a gate that nobody else will open. + status.DrainState.ShouldNotBe(HistorianDrainState.NotPrimary); + status.QueueDepth.ShouldBe(0); + } + else + { + // Held for the peer that shares this queue. + status.DrainState.ShouldBe(HistorianDrainState.NotPrimary); + status.QueueDepth.ShouldBe(1); + } + } + } + finally + { + SqliteConnection.ClearAllPools(); + try { Directory.Delete(tempDir, recursive: true); } catch (IOException) { /* best effort */ } + } + } + + /// A view that always says "some other node is the Primary". + private sealed class StandDownRoleView : IRedundancyRoleView + { + public bool ShouldDrainAlarmHistory => false; + + public void Publish(bool shouldDrainAlarmHistory) { } + } + + private static AlarmHistorianEvent SampleEvent => new( + AlarmId: "eq/alarm-1", + EquipmentPath: "line-1/eq-1", + AlarmName: "temp-high", + AlarmTypeName: "LimitAlarm", + Severity: ZB.MOM.WW.OtOpcUa.Core.Abstractions.AlarmSeverity.High, + EventKind: "Activated", + Message: "temperature high", + User: "system", + Comment: null, + TimestampUtc: new DateTime(2026, 7, 21, 0, 0, 1, DateTimeKind.Utc)); + + /// + /// An enabled historian with no LocalDb registered must fail at resolution, loudly. + /// + /// + /// The queue has nowhere to live without it. Falling back to the Null sink here would look + /// like a working historian while silently discarding every alarm event — the failure mode + /// this whole registration is meant to make impossible. + /// + [Fact] + public void Section_enabled_without_localdb_throws_rather_than_silently_discarding() + { + var services = BaseServices(); + var config = ConfigFrom(new Dictionary { ["AlarmHistorian:Enabled"] = "true" }); + + services.AddAlarmHistorian(config, (_, _) => new FakeWriter()); + + using var provider = services.BuildServiceProvider(); + Should.Throw(() => provider.GetRequiredService()); + } + [Fact] public void Section_binds_drain_capacity_and_retention_knobs() { @@ -112,17 +245,10 @@ public sealed class AlarmHistorianRegistrationTests opts.DeadLetterRetentionDays.ShouldBe(7); } - [Fact] - public void Validate_warns_on_relative_database_path_when_enabled() - { - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "alarm-historian.db" }; - opts.Validate().ShouldContain(w => w.Contains("DatabasePath")); - } - [Fact] public void Validate_is_silent_when_correctly_configured() { - new AlarmHistorianOptions { Enabled = true, DatabasePath = "/abs/h.db" }.Validate().ShouldBeEmpty(); + new AlarmHistorianOptions { Enabled = true }.Validate().ShouldBeEmpty(); } [Fact] @@ -134,39 +260,39 @@ public sealed class AlarmHistorianRegistrationTests [Fact] public void Validate_warns_on_non_positive_drain_interval() { - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "/abs/h.db", DrainIntervalSeconds = 0 }; + var opts = new AlarmHistorianOptions { Enabled = true, DrainIntervalSeconds = 0 }; opts.Validate().ShouldContain(w => w.Contains("DrainIntervalSeconds")); } [Fact] public void Validate_warns_on_non_positive_capacity() { - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "/abs/h.db", Capacity = 0 }; + var opts = new AlarmHistorianOptions { Enabled = true, Capacity = 0 }; opts.Validate().ShouldContain(w => w.Contains("Capacity")); } [Fact] public void Validate_warns_on_non_positive_retention() { - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "/abs/h.db", DeadLetterRetentionDays = 0 }; + var opts = new AlarmHistorianOptions { Enabled = true, DeadLetterRetentionDays = 0 }; opts.Validate().ShouldContain(w => w.Contains("DeadLetterRetentionDays")); } [Fact] public void Validate_warns_on_non_positive_max_attempts() { - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "/abs/h.db", MaxAttempts = 0 }; + var opts = new AlarmHistorianOptions { Enabled = true, MaxAttempts = 0 }; opts.Validate().ShouldContain(w => w.Contains("MaxAttempts")); } [Fact] public void Validate_accumulates_multiple_warnings() { - // relative path + non-positive drain interval ⇒ both warnings, not short-circuited on the first. - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "alarm-historian.db", DrainIntervalSeconds = 0 }; + // Two bad knobs ⇒ both warnings, not short-circuited on the first. + var opts = new AlarmHistorianOptions { Enabled = true, DrainIntervalSeconds = 0, Capacity = 0 }; var warnings = opts.Validate(); - warnings.ShouldContain(w => w.Contains("DatabasePath")); warnings.ShouldContain(w => w.Contains("DrainIntervalSeconds")); + warnings.ShouldContain(w => w.Contains("Capacity")); warnings.Count.ShouldBeGreaterThanOrEqualTo(2); } }