From 124de57e6fe2aaa1a05a9318facabf8751e06f9f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 03:56:06 -0400 Subject: [PATCH 01/13] docs(localdb): phase-2 recon findings Answers Task 0 with file:line citations against master 2e46d054. The STOP condition does not fire: PayloadJson is TEXT, no BLOB column, so alarm_sf_events can be registered for replication as-is. Two findings reshape the plan's assumptions: - The drain worker is not a hosted service or an actor. It is a self-rescheduling Timer owned by the sink and started from the DI factory, so the Primary gate has to live inside DrainOnceAsync as an injected delegate -- Core.AlarmHistorian cannot reference PrimaryGatePolicy, which lives in Runtime. - A Primary gate already exists on the enqueue side (HistorianAdapterActor.ShouldHistorize), using a different policy than the one the plan specifies for the drain. That gate is why today's ungated drain is safe: the Secondary never enqueues, so its queue is empty. Replicating the table breaks that invariant, which makes the drain gate mandatory in the same commit rather than a refinement. Records deviations D-1 (deterministic content-hash ids so a boot-window double-enqueue converges instead of duplicating), D-2 (gate denial must be observable or a snapshot identity mismatch silently stops alarm history), D-3 (Tasks 2+3 land as one commit) and D-4 (the rig will dead-letter in ~5 minutes at MaxAttempts 10 with no gateway). Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- ...7-20-localdb-adoption-phase2.md.tasks.json | 83 +++++- docs/plans/2026-07-20-localdb-phase2-recon.md | 257 ++++++++++++++++++ 2 files changed, 330 insertions(+), 10 deletions(-) create mode 100644 docs/plans/2026-07-20-localdb-phase2-recon.md diff --git a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json index 7cbfdc9a..2f31a474 100644 --- a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json +++ b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json @@ -1,15 +1,78 @@ { "planPath": "docs/plans/2026-07-20-localdb-adoption-phase2.md", "tasks": [ - { "id": 0, "subject": "Task 0: Recon — sink schema, seam, drain lifecycle, role-view bridge (STOP conditions)", "status": "pending" }, - { "id": 1, "subject": "Task 1: alarm_sf_events schema + registration (+ exact-set pin update)", "status": "pending", "blockedBy": [0] }, - { "id": 2, "subject": "Task 2: Rewire sink onto ILocalDb + delete bespoke file management (cutover 1/2)", "status": "pending", "blockedBy": [1] }, - { "id": 3, "subject": "Task 3: Primary-gated drain via PrimaryGatePolicy (cutover 2/2 — may co-commit with Task 2)", "status": "pending", "blockedBy": [2] }, - { "id": 4, "subject": "Task 4: One-time alarm-historian.db legacy migrator", "status": "pending", "blockedBy": [3] }, - { "id": 5, "subject": "Task 5: Convergence + failover scenarios in the pair harness (+ positive control)", "status": "pending", "blockedBy": [3] }, - { "id": 6, "subject": "Task 6: Rig config + docs", "status": "pending", "blockedBy": [3] }, - { "id": 7, "subject": "Task 7: DoD sweep (offline) — STOP and report after this", "status": "pending", "blockedBy": [4, 5, 6] }, - { "id": 8, "subject": "Task 8: Live gate on the docker-dev rig (needs explicit user go-ahead)", "status": "pending", "blockedBy": [7] } + { + "id": 0, + "subject": "Task 0: Recon \u2014 sink schema, seam, drain lifecycle, role-view bridge (STOP conditions)", + "status": "completed", + "note": "STOP condition does NOT fire (no BLOB; PayloadJson TEXT). Recon doc: docs/plans/2026-07-20-localdb-phase2-recon.md. Key finding: the drain worker is an internal Timer inside the sink (not a hosted service/actor), so the gate is a Func ctor param; and HistorianAdapterActor already primary-gates ENQUEUE with a different policy (ShouldHistorize) - which is why today's ungated drain is safe and why replication breaks it. Deviations D-1..D-4 recorded." + }, + { + "id": 1, + "subject": "Task 1: alarm_sf_events schema + registration (+ exact-set pin update)", + "status": "pending", + "blockedBy": [ + 0 + ] + }, + { + "id": 2, + "subject": "Task 2: Rewire sink onto ILocalDb + delete bespoke file management (cutover 1/2)", + "status": "pending", + "blockedBy": [ + 1 + ] + }, + { + "id": 3, + "subject": "Task 3: Primary-gated drain via PrimaryGatePolicy (cutover 2/2 \u2014 may co-commit with Task 2)", + "status": "pending", + "blockedBy": [ + 2 + ] + }, + { + "id": 4, + "subject": "Task 4: One-time alarm-historian.db legacy migrator", + "status": "pending", + "blockedBy": [ + 3 + ] + }, + { + "id": 5, + "subject": "Task 5: Convergence + failover scenarios in the pair harness (+ positive control)", + "status": "pending", + "blockedBy": [ + 3 + ] + }, + { + "id": 6, + "subject": "Task 6: Rig config + docs", + "status": "pending", + "blockedBy": [ + 3 + ] + }, + { + "id": 7, + "subject": "Task 7: DoD sweep (offline) \u2014 STOP and report after this", + "status": "pending", + "blockedBy": [ + 4, + 5, + 6 + ] + }, + { + "id": 8, + "subject": "Task 8: Live gate on the docker-dev rig (needs explicit user go-ahead)", + "status": "pending", + "blockedBy": [ + 7 + ] + } ], - "lastUpdated": "2026-07-20T00:00:00Z" + "lastUpdated": "2026-07-21T00:00:00Z" } diff --git a/docs/plans/2026-07-20-localdb-phase2-recon.md b/docs/plans/2026-07-20-localdb-phase2-recon.md new file mode 100644 index 00000000..97fad21b --- /dev/null +++ b/docs/plans/2026-07-20-localdb-phase2-recon.md @@ -0,0 +1,257 @@ +# LocalDb Phase 2 — Recon findings + +**Date:** 2026-07-21 · **Branch:** `feat/localdb-phase2` · **Base:** `master` `2e46d054` + +Answers Task 0 of `2026-07-20-localdb-adoption-phase2.md`. Every claim carries a `file:line` +citation against the base commit. Read the **Deviations** section last — two of them change the +shape of Tasks 2/3 from what the plan assumed. + +--- + +## 1. The sink's table schema + +**STOP condition does NOT fire** — confirmed by re-reading the executed DDL, as the plan's +pre-answer instructed. + +Executed DDL: `SqliteStoreAndForwardSink.cs:652-670` (`InitializeSchema`). It matches the class +doc-comment at `:17-26` exactly — no drift between documented and executed schema. + +```sql +CREATE TABLE IF NOT EXISTS Queue ( + RowId INTEGER PRIMARY KEY AUTOINCREMENT, + AlarmId TEXT NOT NULL, + EnqueuedUtc TEXT NOT NULL, + PayloadJson TEXT NOT NULL, + AttemptCount INTEGER NOT NULL DEFAULT 0, + LastAttemptUtc TEXT NULL, + LastError TEXT NULL, + DeadLettered INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS IX_Queue_Drain ON Queue (DeadLettered, RowId); +``` + +| Question | Answer | +|---|---| +| BLOB columns? | **None.** All 8 columns are TEXT/INTEGER. Payload is `PayloadJson TEXT NOT NULL`. No base64 conversion needed; the 171 KB chunk sizing is moot. | +| PK autoincrement? | **Yes** — `RowId INTEGER PRIMARY KEY AUTOINCREMENT`. The migrator must mint deterministic `mig-{node}-{legacyId}` ids and the new table needs a TEXT PK, as the plan specified. | +| Indices to carry | One: `IX_Queue_Drain (DeadLettered, RowId)` — the drain's covering index. `alarm_sf_events` wants the same shape over (status, insertion order). | +| Largest realistic payload | `PayloadJson` is a serialized `AlarmHistorianEvent` — 10 scalar fields (`AlarmHistorianEvent.cs`), no collections, no nesting. Low single-digit KB worst case; `Comment` is the only loosely-bounded field. | + +**Legacy column list** for the migrator's `pragma_table_info` intersection check: +`RowId, AlarmId, EnqueuedUtc, PayloadJson, AttemptCount, LastAttemptUtc, LastError, DeadLettered`. + +`RegisterReplicated` will accept this table: no BLOB, and the PK will be app-minted TEXT. + +## 2. The public seam + +`IAlarmHistorianSink` (`IAlarmHistorianSink.cs:23-34`) is **two members**: + +```csharp +Task EnqueueAsync(AlarmHistorianEvent evt, CancellationToken cancellationToken); +HistorianSinkStatus GetStatus(); +``` + +The concrete `SqliteStoreAndForwardSink` additionally exposes, and these are all consumed +somewhere, so the rewire must keep them: + +| Member | Cited | Consumer | +|---|---|---| +| `StartDrainLoop(TimeSpan)` | `:189` | `ServiceCollectionExtensions.cs:105` (DI factory) | +| `DrainOnceAsync(CancellationToken)` | `:320` | tests only — the deterministic drive | +| `RetryDeadLettered()` | `:511` | operator action (AdminUI) | +| `CurrentBackoff` | `:676` | tests | +| `DebugCapacityProbeCount` | `:99` | tests | +| `DefaultCapacity` / `DefaultMaxAttempts` consts | `:49`,`:53` | `AlarmHistorianOptions.cs:38,45` | +| `IDisposable` | `:679` | DI + tests; **disposes the injected writer too** | + +There is **no separate enqueue/dequeue/markDelivered/deadLetter interface** — the plan's item 2 +guessed at one. Dequeue, delivered-marking and dead-lettering are all *private* to the sink +(`ReadBatch :532`, `DeleteRow :564`, `DeadLetterRow :573`, `BumpAttempt :587`). That is good news: +the rewire is entirely internal to one class and the public seam is untouched. + +## 3. Semantics to preserve + +All defaults live in **two** places that must stay in agreement: + +| Semantic | Sink default | Options default | Enforced at | +|---|---|---|---| +| `Capacity` 1,000,000 | `:49` | `AlarmHistorianOptions.cs:38` | `EnforceCapacityFastPathAsync :271` → `EnforceCapacityAsync :602` | +| `MaxAttempts` 10 | `:53` | `:45` | drain loop `:429` | +| Dead-letter retention 30 d | `:50` | `:41` | `PurgeAgedDeadLetters :638`, run once per drain tick | +| `BatchSize` 100 | ctor arg `:116` | `:31` | `ReadBatch :542` | +| `DrainIntervalSeconds` 5 | — | `:34` | `StartDrainLoop` via `ServiceCollectionExtensions.cs:105` | + +Behaviours that are **not** in the plan's list but are load-bearing, and which the cutover must +not silently drop: + +- **Backoff ladder** 1s→2s→5s→15s→60s (`:57-64`), applied to the timer's *next due time* + (`RescheduleDrain :224`) so an outage genuinely slows the cadence. +- **Overflow evicts oldest non-dead-lettered rows** with a WARN and a lifetime + `EvictedCount` counter (`:630-635`) — the durability guarantee is explicitly bounded. +- **Corrupt-payload rows dead-letter immediately** for their own RowId so they cannot stall the + queue head (`:344-361`). +- **Cardinality guard**: a writer returning ≠1 outcome per event is treated as a batch retry + (`:393-404`). +- **In-memory row counter with periodic resync** every 10,000 enqueues (`:89-96`, `:271-290`) — + a perf optimisation that avoids `COUNT(*)` on the hot enqueue path. +- **Post-dispose throwing contract**: `GetStatus`/`RetryDeadLettered`/`EnqueueAsync`/`StartDrainLoop` + throw `ObjectDisposedException`; `DrainOnceAsync` returns quietly (`:322`). Pinned by a + regression test (`SqliteStoreAndForwardSinkTests.cs:854-865`). + +## 4. Drain-worker lifecycle — **the finding that reshapes Task 3** + +> The plan asked "hosted service or actor?". **Neither.** + +The drain worker is a **self-rescheduling one-shot `System.Threading.Timer` owned by the sink +itself** (`_drainTimer`, `:76`; started by `StartDrainLoop :189`; callback `DrainTimerCallback :199`; +re-armed by `RescheduleDrain :224`). It is started from inside the DI singleton factory at +`ServiceCollectionExtensions.cs:105`, so it begins ticking the moment the first consumer resolves +`IAlarmHistorianSink`. + +Consequences for the Primary gate: + +1. **The gate has to live inside `DrainOnceAsync`**, not in a wrapper. There is no external + scheduler to gate. +2. **`Core.AlarmHistorian` cannot see `PrimaryGatePolicy`.** The policy is in + `Runtime/Drivers/PrimaryGatePolicy.cs`, and Runtime → Core is the reference direction. + `Core.AlarmHistorian` references only `Core.Abstractions` (`.csproj`). So the sink takes a + **`Func` gate delegate** (default: always-drain) and Runtime supplies the lambda that + calls `PrimaryGatePolicy`. No new project reference, no new interface in Core. +3. **A role view is still needed** because the timer fires outside any actor. See the next point + for who should own it. + +### There is already a Primary gate — on the *enqueue* side, with a different policy + +`HistorianAdapterActor` (`Historian/HistorianAdapterActor.cs`) already subscribes to +`RedundancyStateChanged` (`:86`, handler `:145`), caches `_localRole` (`:42`), and gates the +enqueue on `ShouldHistorize()` (`:97`): + +```csharp +private bool ShouldHistorize() => _localRole is not (RedundancyRole.Secondary or RedundancyRole.Detached); +``` + +This is **not** `PrimaryGatePolicy` — it ignores driver-member count and defaults to *write* while +the role is unknown. Its comment (`:68-71`) explains why it exists: DPS fans the Primary's single +`alerts` publish to **every** node's adapter, so without the gate both nodes of a pair would +double-write. + +**This is why today's ungated drain is safe, and why Phase 2 breaks that.** Today the Secondary +never enqueues, so its queue is empty and an ungated drain has nothing to send. Once +`alarm_sf_events` replicates, the Primary's rows land in the Secondary's table — and an ungated +drain there would double-deliver *every* event, continuously, not just at failover. **The drain +gate is not a nicety; it is required by the same commit that registers the table.** + +### Who owns the role view + +`HistorianAdapterActor`, not `DriverHostActor`. Both are spawned under +`WithOtOpcUaRuntimeActors` (`ServiceCollectionExtensions.cs:353`) which `Program.cs:133` calls only +under `hasDriver` — as are `AddAlarmHistorian` (`:175`) and `AddOtOpcUaLocalDb` (`:185`). So sink, +LocalDb and adapter are co-located by construction. The adapter wins over `DriverHostActor` +because it **already** holds the role for exactly this purpose; using it adds no new subscription +and no new way for the view to go stale while the sink keeps draining. + +`DriverMemberCount` is read off Akka cluster state in `DriverHostActor.DriverMemberCount() :1446` +(try/catch → 0 on a non-cluster provider ⇒ single-node posture). Task 3 will lift that into a +small shared helper rather than copy it. + +## 5. Other `DatabasePath` construction sites + +- `ServiceCollectionExtensions.cs:98` — the only production site. +- `appsettings.json:57` — `"DatabasePath": "alarm-historian.db"`. +- `AlarmHistorianOptions.Validate() :54-55` — warns when the path is relative. **This warning + disappears with the key**; the validator loses one of its five checks. +- `SqliteStoreAndForwardSinkTests.cs` — ~30 construction sites, all passing the per-test temp + `_dbPath` seeded at `:25` and deleted in `Dispose :32`. These are the tests Task 2 rewires onto + a temp-file `ILocalDb`. +- **`docker-dev/docker-compose.yml` has no `AlarmHistorian` keys at all** — the rig runs with + `Enabled=false` today. Task 6 is therefore an *addition*, not an edit. + +No tooling or CLI constructs the sink. + +## 6. How `Enabled=false` short-circuits + +`AddAlarmHistorian` returns before registering anything when the section is absent or +`Enabled != true` (`ServiceCollectionExtensions.cs:87`), leaving the `NullAlarmHistorianSink` +seeded by `AddOtOpcUaRuntime :49`. The Null sink is a pure no-op (`IAlarmHistorianSink.cs:37-52`). + +Confirming the plan's own answer: **the tables are created unconditionally in `OnReady`** (cheap, +empty) because `AddOtOpcUaLocalDb` is independent of the `AlarmHistorian` gate. Only the +sink/drain stay Null. This is also required for correctness — `RegisterReplicated` must run on +both pair members regardless of their historian config, or a node that later enables the sink +would write rows that are never captured (see the CDC note below). + +## 7. Library constraints re-confirmed (README, `ZB.MOM.WW.LocalDb` 0.1.3) + +- **BLOB columns are rejected at registration.** Not a problem here (§1). +- **CDC semantics** — rows that existed *before* `RegisterReplicated` are neither captured nor + snapshotted. This is the mechanical reason the migrator must run **after** registration, and + the reason `LocalDbSetup.OnReady`'s ordering comment is load-bearing (`LocalDbSetup.cs:23-31`). +- **LWW keyed on the PK** — two nodes minting the same PK for *different* rows silently overwrite + each other; two nodes minting the same PK for the *same* row converge. Decision D-1 below turns + that second property into a feature. +- `ILocalDb` surface in use: `ExecuteAsync` / `QueryAsync(sql, rowMapper, params, ct)` / + `BeginTransactionAsync` / `CreateConnection` / `RegisterReplicated` + (`Deployment/LocalDbDeploymentArtifactCache.cs:77,82,205`). + +--- + +## Deviations and decisions + +Recorded here rather than raised as questions, per the plan's follow-up convention. + +### D-1 — Enqueue ids are a deterministic content hash, not `Guid.NewGuid()` + +**The plan says** GUIDs minted at enqueue (Task 2 step 2). + +**The problem.** The enqueue gate (`ShouldHistorize`, default-**write** on unknown role) and the +drain gate the plan specifies (`PrimaryGatePolicy`, default-**deny** on unknown role when paired) +disagree during the boot window. On a two-node pair before the first redundancy snapshot arrives, +**both** adapters enqueue the same DPS-fanned transition. With random GUIDs those are two distinct +rows that replicate into each other's tables — the buffer *duplicates* instead of converging, and +whichever node becomes Primary later delivers both copies. + +**The decision.** Mint `id` as a hash over the serialized payload. Two nodes independently +enqueuing the same fanned event produce the same PK, so LWW converges them to one row for free. +Two genuinely distinct events cannot collide: `AlarmHistorianEvent` carries a full-precision +`TimestampUtc` plus alarm id, kind, message and user, so an identical hash means an identical +event. + +This costs one `SHA256.HashData` per enqueue and removes a whole duplicate-row class. It also +slightly *improves* today's behaviour, where the same boot-window fan-out is already +double-delivered because the drain is ungated on both nodes. + +`ShouldHistorize` itself is left unchanged — out of scope, and with D-1 in place the two gates +disagreeing is no longer harmful. + +### D-2 — Gate denial must be observable + +`PrimaryGatePolicy` defaults to **deny** when the role is unknown on a multi-driver cluster. That +is the correct safety posture, but it means a redundancy-snapshot identity mismatch — the +documented 03/S5 shape that `DriverHostActor.cs:1485-1491` already warns about once — would +silently stop alarm history on *both* nodes, degrading only as retry/eviction growth minutes later. + +Task 3 will therefore: log the skip once per gate-state transition (not per tick — the drain runs +every 5 s), and surface it in `HistorianSinkStatus` so `/healthz` and the AdminUI can see +"buffering, not draining, because not Primary" rather than an unexplained rising queue depth. +Adding a `HistorianDrainState` member is additive; Task 3 will check every `switch` over it. + +### D-3 — Tasks 2 and 3 will land as one commit + +The plan anticipated this. Confirmed necessary: the gate delegate is a constructor parameter of +the rewritten sink, so the sink cannot compile in its final shape without it, and an intermediate +commit with a rewired-but-ungated sink would be a commit in which a replicated table is drained by +both nodes. Not a state worth having in history. + +### D-4 — The live gate needs `MaxAttempts` raised on the rig + +Not actionable until Task 8, recorded now so it is not rediscovered there. The rig has no +HistorianGateway, so every drain attempt fails. With `MaxAttempts: 10` and the backoff ladder +capping at 60 s, buffered rows dead-letter roughly five minutes into the outage — which would make +the gate's "dead letters 0" check (step 2) fail for reasons that have nothing to do with +replication. Task 6 should set a high `MaxAttempts` on the rig, or Task 8 should assert on +dead-letter *timing* instead. + +Related: `Program.cs:170-173` shows an `AlarmHistorian:Enabled=true` / +`ServerHistorian:Enabled=false` deployment is explicitly supported and warned about — which is +exactly the rig's shape. Task 6 must confirm `ServerHistorianOptionsValidator` does not fail host +start for a disabled section with an empty `Endpoint`. From 8a9cb40a720cbd51b6c2b69a944a78417b524e8c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 03:58:50 -0400 Subject: [PATCH 02/13] feat(localdb): alarm_sf_events replicated table Adds the alarm store-and-forward buffer to the consolidated LocalDb file and registers it for replication, so a node that dies with undelivered alarm history no longer takes it to the grave. The table keeps the legacy queue's shape rather than the status column the plan sketched. An acknowledged row is deleted, not marked delivered: that needs no second sweeper to keep the table bounded, leaves the capacity semantics untouched, and the replication engine carries the delete as a tombstone so the peer drops its copy anyway -- which is what the status column was for. last_error is retained because it is the only operator-facing record of why a row was dead-lettered. The primary key is app-minted TEXT. The legacy AUTOINCREMENT RowId cannot replicate under last-writer-wins: two nodes would independently allocate rowid 7 to different alarms and silently overwrite each other. The drain index therefore orders by enqueued_at_utc rather than insertion order, with id as a tiebreak so the ordering is total. Tables are created unconditionally, independent of AlarmHistorian:Enabled. An empty registered table costs three triggers; creating it lazily would mean a node that enables the historian later writes rows before its capture triggers exist, which is exactly the silent-loss shape OnReady's ordering comment warns about. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- ...7-20-localdb-adoption-phase2.md.tasks.json | 5 +- docs/plans/2026-07-20-localdb-phase2-recon.md | 21 +++++ .../AlarmSfSchema.cs | 77 +++++++++++++++++++ .../Configuration/LocalDbSetup.cs | 18 ++++- .../LocalDbSetupTests.cs | 48 +++++++++++- 5 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs diff --git a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json index 2f31a474..db3f2c94 100644 --- a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json +++ b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json @@ -10,10 +10,11 @@ { "id": 1, "subject": "Task 1: alarm_sf_events schema + registration (+ exact-set pin update)", - "status": "pending", + "status": "completed", "blockedBy": [ 0 - ] + ], + "note": "alarm_sf_events created in AlarmSfSchema (Core.AlarmHistorian) + registered third in LocalDbSetup.OnReady. Exact-set pin updated to 3 tables. DEVIATION D-5: kept the legacy delete-on-ack + dead_lettered flag + last_error rather than the plan's status column (no sweeper for 'delivered' rows; last_error is the only record of why a row died). Drain ORDER BY moves to (enqueued_at_utc, id)." }, { "id": 2, diff --git a/docs/plans/2026-07-20-localdb-phase2-recon.md b/docs/plans/2026-07-20-localdb-phase2-recon.md index 97fad21b..f5714fba 100644 --- a/docs/plans/2026-07-20-localdb-phase2-recon.md +++ b/docs/plans/2026-07-20-localdb-phase2-recon.md @@ -242,6 +242,27 @@ the rewritten sink, so the sink cannot compile in its final shape without it, an commit with a rewired-but-ungated sink would be a commit in which a replicated table is drained by both nodes. Not a state worth having in history. +### D-5 — The table keeps the legacy delete-on-ack shape, not a `status` column + +**The plan proposes** `status TEXT NOT NULL DEFAULT 'pending'` with values `pending | delivered | +dead`, and drops the legacy `LastError` column. + +**Two problems.** A `delivered` status means acknowledged rows accumulate in the table forever — +the plan specifies no sweeper for them, and they would also have to be excluded from the capacity +count, changing a semantic §3 says to preserve. And `LastError` is not dead weight: `DeadLetterRow` +writes the failure reason into it and `RetryDeadLettered` clears it, so it is the only operator- +facing record of *why* a row was dead-lettered. + +**The decision.** Keep the legacy shape: DELETE on acknowledgement, `dead_lettered` as the same 0/1 +flag, and retain `last_error`. Deleting keeps the table bounded with no second sweeper, and the +replication engine carries the delete as a tombstone (pruned after `TombstoneRetention`, 7 d), so +the peer drops its copy too — which is the property the plan wanted the `delivered` status for. +Columns then map 1:1 onto the legacy table, making the Task-4 migrator a straight copy. + +One genuine change: the drain's `ORDER BY` moves from `RowId ASC` to `enqueued_at_utc, id`, because +a hashed TEXT id carries no insertion order. Round-trip ("O") timestamps sort lexicographically in +chronological order, and `id` makes the ordering total; the index covers both. + ### D-4 — The live gate needs `MaxAttempts` raised on the rig Not actionable until Task 8, recorded now so it is not rediscovered there. The rig has no diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs new file mode 100644 index 00000000..eaf4553c --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs @@ -0,0 +1,77 @@ +using Microsoft.Data.Sqlite; + +namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; + +/// +/// DDL for the alarm store-and-forward buffer: one row per alarm event awaiting delivery to +/// the historian gateway. +/// +/// +/// +/// Replaces the standalone alarm-historian.db the sink used to own outright. Living +/// in the consolidated LocalDb file is what lets the buffer replicate to the redundant pair +/// peer, so a node that dies with undelivered alarm history no longer takes it to the grave. +/// +/// +/// Deliberately depends on nothing but so it can be applied +/// to any connection — the host's LocalDbSetup.OnReady in production, and a bare +/// connection in a test — without dragging the DI graph along. Mirrors +/// DeploymentCacheSchema. +/// +/// +/// Why is TEXT and app-minted. The legacy queue keyed on +/// RowId INTEGER PRIMARY KEY AUTOINCREMENT. Convergence is last-writer-wins over the +/// primary key, so two nodes independently allocating rowid 7 for different alarms would +/// silently overwrite one another. The sink mints the id from a hash of the payload, which +/// additionally makes the same event converge to one row when both nodes of a pair enqueue +/// it — as they legitimately do in the window before the first redundancy snapshot arrives. +/// +/// +public static class AlarmSfSchema +{ + /// Table holding queued alarm events awaiting delivery. + public const string EventsTable = "alarm_sf_events"; + + /// The single primary-key column. + public const string IdColumn = "id"; + + /// + /// Creates the buffer table if it does not already exist. Idempotent. + /// + /// + /// An already-open connection. ILocalDb.CreateConnection() hands out open, + /// pragma-configured connections — do not call Open() on one. + /// + public static void Apply(SqliteConnection connection) + { + ArgumentNullException.ThrowIfNull(connection); + + using var cmd = connection.CreateCommand(); + + // Columns map 1:1 onto the legacy Queue table so the one-time migrator is a straight copy + // and the drain keeps its existing semantics: dead_lettered is the same 0/1 flag, and an + // acknowledged row is DELETEd rather than marked. Deleting keeps the table bounded without + // a second sweeper, and the replication engine carries the delete as a tombstone, so the + // peer drops its copy too. + // + // The drain index orders by enqueued_at_utc because a hashed TEXT id carries no insertion + // order the way the legacy AUTOINCREMENT RowId did. Timestamps are round-trip ("O") format, + // so lexicographic ordering is chronological; id breaks ties so the order is total and the + // index covers the drain's read. + cmd.CommandText = """ + CREATE TABLE IF NOT EXISTS alarm_sf_events ( + id TEXT NOT NULL PRIMARY KEY, + alarm_id TEXT NOT NULL, + enqueued_at_utc TEXT NOT NULL, + payload_json TEXT NOT NULL, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_attempt_utc TEXT NULL, + last_error TEXT NULL, + dead_lettered INTEGER NOT NULL DEFAULT 0 + ); + CREATE INDEX IF NOT EXISTS ix_alarm_sf_events_drain + ON alarm_sf_events (dead_lettered, enqueued_at_utc, id); + """; + cmd.ExecuteNonQuery(); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs index 0ce2d3be..2819ea17 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs @@ -1,11 +1,12 @@ using ZB.MOM.WW.LocalDb; +using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache; namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; /// -/// The onReady callback handed to AddZbLocalDb: creates the deployment-cache -/// tables and opts them into replication. +/// The onReady callback handed to AddZbLocalDb: creates the deployment-cache and +/// alarm store-and-forward tables and opts them into replication. /// /// /// @@ -26,8 +27,15 @@ public static class LocalDbSetup /// RegisterReplicated is what installs the three AFTER triggers that capture /// changes into the oplog. Any row written before that call is never captured, so it /// never reaches the peer — silently, and permanently, because nothing ever revisits - /// history. Phase 1 writes nothing here; when Phase 2 adds its store-and-forward - /// migrator, the migrator must run after both registrations for the same reason. + /// history. The Phase-2 legacy alarm migrator therefore runs after every + /// registration, not alongside the DDL. + /// + /// + /// The alarm buffer's tables are created unconditionally, regardless of whether this + /// node has AlarmHistorian:Enabled set. An empty registered table costs three + /// triggers and nothing else, whereas creating it lazily when the sink first appears + /// would mean a node that enables the historian later writes rows before its triggers + /// exist — which is precisely the silent-loss shape above. /// /// /// The freshly constructed local database. @@ -40,9 +48,11 @@ public static class LocalDbSetup using (var connection = db.CreateConnection()) { DeploymentCacheSchema.Apply(connection); + AlarmSfSchema.Apply(connection); } db.RegisterReplicated(DeploymentCacheSchema.ArtifactsTable); db.RegisterReplicated(DeploymentCacheSchema.PointerTable); + db.RegisterReplicated(AlarmSfSchema.EventsTable); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs index 2db014c2..77481b9b 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs @@ -46,7 +46,7 @@ public sealed class LocalDbSetupTests : IDisposable } [Fact] - public void OnReady_RegistersExactlyTheTwoDeploymentTables() + public void OnReady_RegistersExactlyTheThreeReplicatedTables() { // Both directions are load-bearing. "No fewer" catches a dropped RegisterReplicated call // (that table then never replicates). "No more" catches an accidental registration — @@ -54,7 +54,7 @@ public sealed class LocalDbSetupTests : IDisposable var db = BuildDb(); db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal) - .ShouldBe(["deployment_artifacts", "deployment_pointer"]); + .ShouldBe(["alarm_sf_events", "deployment_artifacts", "deployment_pointer"]); } [Fact] @@ -78,6 +78,50 @@ public sealed class LocalDbSetupTests : IDisposable db.ReplicatedTables["deployment_pointer"].PkColumns.ShouldBe(["cluster_id"]); } + [Fact] + public void AlarmSfEvents_PkIsTheAppMintedId() + { + // A single TEXT PK the application mints. The legacy queue this table replaces keyed on + // `RowId INTEGER PRIMARY KEY AUTOINCREMENT`, which LWW cannot replicate: two nodes would + // independently allocate rowid 7 for different alarms and silently overwrite each other. + var db = BuildDb(); + + db.ReplicatedTables["alarm_sf_events"].PkColumns.ShouldBe(["id"]); + } + + [Fact] + public async Task AlarmSfEventRows_EnterTheOplog() + { + // Same ordering assertion as the deployment-pointer test below, for the table Phase 2 adds. + // Worth its own case because the alarm table is registered by a different call site, and a + // registration added ahead of its DDL would fail loudly while one added after the migrator + // would fail silently. + var db = BuildDb(); + + await db.ExecuteAsync( + """ + INSERT INTO alarm_sf_events + (id, alarm_id, enqueued_at_utc, payload_json, attempt_count) + VALUES (@Id, @AlarmId, @EnqueuedAtUtc, @PayloadJson, 0) + """, + new + { + Id = new string('c', 64), + AlarmId = "equip/alarm-1", + EnqueuedAtUtc = "2026-07-21T00:00:00.0000000Z", + PayloadJson = """{"AlarmId":"equip/alarm-1"}""", + }, + TestContext.Current.CancellationToken); + + var oplogRows = await db.QueryAsync( + "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'alarm_sf_events'", + r => r.GetInt32(0), + parameters: null, + TestContext.Current.CancellationToken); + + oplogRows[0].ShouldBe(1); + } + [Fact] public async Task RowsWrittenAfterOnReady_EnterTheOplog() { From 71379816e74ebafe065fe414ad6bbf3724231f07 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 04:13:00 -0400 Subject: [PATCH 03/13] feat(localdb): alarm S&F on LocalDb with primary-gated drain (cutover) Moves the alarm store-and-forward buffer out of its own alarm-historian.db and into the node's consolidated LocalDb, where it replicates to the redundant pair peer. A node that dies holding undelivered alarm history no longer takes it to the grave. Tasks 2 and 3 land together, as the plan anticipated. They are not separable: the gate is a constructor argument of the rewritten sink, and a commit that replicated the queue without gating the drain would be a commit in which both nodes of a pair deliver every alarm event, continuously. That drain gate is the load-bearing part of the change, and the recon explains why it is new work rather than a refinement. Exactly-once delivery across a pair is enforced today on the ENQUEUE side, by HistorianAdapterActor: only the Primary enqueues, so the Secondary's queue is empty and its ungated drain has nothing to send. Replicating the table destroys that invariant. The gate is a Func the caller supplies, because the drain runs on a timer the sink owns rather than on a mailbox, and Core.AlarmHistorian cannot reference PrimaryGatePolicy in Runtime. Runtime supplies it via a new IRedundancyRoleView singleton that DriverHostActor publishes its Primary-gate verdict to -- the same verdict the inbound-write and native-ack gates use, so there is no second notion of am-I-the-Primary to drift. Two failure modes are deliberately closed: - The view is seeded OPEN, matching the policy's own answer for an unknown role with no driver peer. A deployment that runs no redundancy never publishes to it, and defaulting closed would silently stop its alarm history forever. - A gate that throws is read as not-now, never as permission, and a closed gate reports the new HistorianDrainState.NotPrimary rather than Idle. A Secondary's rising queue is supposed to look different from a stalled drain, and if BOTH nodes report NotPrimary the pair is misconfigured and says so instead of quietly filling toward the capacity ceiling. Row ids are a hash of the payload rather than fresh GUIDs. Both adapters accept the same fanned transition in the window before the first redundancy snapshot arrives, and under last-writer-wins an equal key converges those two accepts into one row instead of duplicating them. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- Directory.Packages.props | 1 + ...7-20-localdb-adoption-phase2.md.tasks.json | 10 +- .../IAlarmHistorianSink.cs | 16 +- ...dSink.cs => LocalDbStoreAndForwardSink.cs} | 522 ++++++++++-------- ....MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj | 6 + .../ZB.MOM.WW.OtOpcUa.Host/appsettings.json | 3 +- .../Drivers/DriverHostActor.cs | 30 +- .../Historian/AlarmHistorianOptions.cs | 21 +- .../Redundancy/IRedundancyRoleView.cs | 46 ++ .../ServiceCollectionExtensions.cs | 42 +- ....cs => LocalDbStoreAndForwardSinkTests.cs} | 266 ++++++--- ...W.OtOpcUa.Core.AlarmHistorian.Tests.csproj | 8 + .../Drivers/DriverHostActorRoleViewTests.cs | 132 +++++ .../AlarmHistorianRegistrationTests.cs | 66 ++- 14 files changed, 804 insertions(+), 365 deletions(-) rename src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/{SqliteStoreAndForwardSink.cs => LocalDbStoreAndForwardSink.cs} (55%) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs rename tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/{SqliteStoreAndForwardSinkTests.cs => LocalDbStoreAndForwardSinkTests.cs} (77%) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 82221614..9754f764 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -57,6 +57,7 @@ + diff --git a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json index db3f2c94..0272fcc7 100644 --- a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json +++ b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json @@ -19,18 +19,20 @@ { "id": 2, "subject": "Task 2: Rewire sink onto ILocalDb + delete bespoke file management (cutover 1/2)", - "status": "pending", + "status": "completed", "blockedBy": [ 1 - ] + ], + "note": "Sink rewritten as LocalDbStoreAndForwardSink over ILocalDb; bespoke file/pragma/schema management deleted with the old class. AlarmHistorian:DatabasePath removed (breaking config key). Ids are a deterministic payload hash (D-1), not GUIDs." }, { "id": 3, "subject": "Task 3: Primary-gated drain via PrimaryGatePolicy (cutover 2/2 \u2014 may co-commit with Task 2)", - "status": "pending", + "status": "completed", "blockedBy": [ 2 - ] + ], + "note": "Drain gated on IRedundancyRoleView, a singleton DriverHostActor publishes PrimaryGatePolicy's verdict to on every snapshot. Fails closed on a throwing gate; seeded OPEN so a non-redundant deployment is never silently stopped. New HistorianDrainState.NotPrimary + transition-logged (D-2). Landed with Task 2 as one commit (D-3)." }, { "id": 4, diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/IAlarmHistorianSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/IAlarmHistorianSink.cs index 7efaf3c9..72287399 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/IAlarmHistorianSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/IAlarmHistorianSink.cs @@ -4,13 +4,13 @@ namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; /// The historian sink contract — where qualifying alarm events land. Ingestion routes /// through the HistorianGateway alarm writer (the gateway's SendEvent gRPC path) /// behind the durable store-and-forward queue. Tests use an in-memory fake; production uses -/// . +/// . /// /// /// /// is fire-and-forget from the engine's perspective — /// the sink MUST NOT block the emitting thread. Production implementations -/// () persist to a local SQLite queue +/// () persist to the node's local SQLite queue /// first, then drain asynchronously to the actual historian. Per the Phase 7 plan, /// failed downstream writes replay with exponential backoff; /// operator actions are never blocked waiting on the historian. @@ -79,6 +79,18 @@ public enum HistorianDrainState Idle, Draining, BackingOff, + + /// + /// Ticking, but not draining: this node does not hold the Primary role, so it leaves the + /// (replicated) queue for the node that does. + /// + /// + /// Distinct from so a rising queue depth on a Secondary reads as the + /// designed behaviour rather than a stalled drain. It is also how a misconfigured pair + /// becomes diagnosable: if BOTH nodes report this, no one is draining and alarm history is + /// silently accumulating toward the capacity ceiling. + /// + NotPrimary, } /// Returned by the historian alarm writer per event — drain worker uses this to decide retry cadence. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/SqliteStoreAndForwardSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs similarity index 55% rename from src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/SqliteStoreAndForwardSink.cs rename to src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs index a76bd1ee..17285524 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/SqliteStoreAndForwardSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs @@ -1,53 +1,56 @@ +using System.Security.Cryptography; +using System.Text; using System.Text.Json; -using Microsoft.Data.Sqlite; using Serilog; +using ZB.MOM.WW.LocalDb; namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; /// -/// Durable SQLite queue on the node absorbs every qualifying alarm event, a drain -/// worker batches rows to the Wonderware historian sidecar via -/// on an exponential-backoff cadence, and -/// operator acks never block on the historian being reachable. +/// Durable queue in the node's consolidated LocalDb absorbs every qualifying alarm event, a +/// drain worker batches rows to the historian gateway via +/// on an exponential-backoff cadence, and operator acks never block on the historian being +/// reachable. /// /// /// -/// Queue schema: -/// -/// CREATE TABLE Queue ( -/// RowId INTEGER PRIMARY KEY AUTOINCREMENT, -/// AlarmId TEXT NOT NULL, -/// EnqueuedUtc TEXT NOT NULL, -/// PayloadJson TEXT NOT NULL, -/// AttemptCount INTEGER NOT NULL DEFAULT 0, -/// LastAttemptUtc TEXT NULL, -/// LastError TEXT NULL, -/// DeadLettered INTEGER NOT NULL DEFAULT 0 -/// ); -/// -/// Dead-lettered rows stay in place for the configured retention window (default -/// 30 days) so operators can inspect + manually -/// retry before the sweeper purges them. Regular queue capacity is bounded — -/// overflow evicts the oldest non-dead-lettered rows with a WARN log. The -/// durability guarantee is therefore bounded by : -/// under a sustained historian outage, accepted events may be evicted before -/// delivery. The counter makes -/// overflow visible to operators without requiring the WARN log to be scraped. +/// Rows live in alarm_sf_events (see ), which the host +/// registers for replication. That is the point of this type living on +/// rather than owning its own file: the buffer mirrors to the +/// redundant pair peer, so a node that dies holding undelivered alarm history no longer +/// takes it to the grave. +/// +/// +/// Only one node of a pair may drain. Replication puts the Primary's queued rows in +/// the Secondary's table too; an ungated drain there would re-deliver every event, +/// continuously. The drainGate constructor argument is how the caller scopes +/// draining to the Primary. Enqueue is gated separately and upstream, by +/// HistorianAdapterActor. +/// +/// +/// Dead-lettered rows stay in place for the configured retention window (default 30 days) +/// so operators can inspect + manually retry before the sweeper purges them. Regular queue +/// capacity is bounded — overflow evicts the oldest non-dead-lettered rows with a WARN log. +/// The durability guarantee is therefore bounded by : under a +/// sustained historian outage, accepted events may be evicted before delivery. The +/// counter makes overflow visible to +/// operators without requiring the WARN log to be scraped. /// /// /// Drain runs on a self-rescheduling one-shot . /// Exponential backoff on : -/// 1s → 2s → 5s → 15s → 60s cap — the backoff is applied to the timer's next -/// due-time, so a historian outage genuinely slows the drain cadence. -/// rows flip -/// the DeadLettered flag on the individual row; neighbors in the batch -/// still retry on their own cadence. +/// 1s → 2s → 5s → 15s → 60s cap — the backoff is applied to the timer's next due-time, so a +/// historian outage genuinely slows the drain cadence. +/// rows flip the dead_lettered flag +/// on the individual row; neighbors in the batch still retry on their own cadence. /// /// -public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable +public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposable { /// Default queue capacity — oldest non-dead-lettered rows evicted past this. public const long DefaultCapacity = 1_000_000; + + /// Default window dead-lettered rows are retained for before the sweeper purges them. public static readonly TimeSpan DefaultDeadLetterRetention = TimeSpan.FromDays(30); /// Default max delivery attempts before a perpetually-retrying (poison) row is dead-lettered. @@ -62,7 +65,7 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable TimeSpan.FromSeconds(60), ]; - private readonly string _connectionString; + private readonly ILocalDb _db; private readonly IAlarmHistorianWriter _writer; private readonly ILogger _logger; private readonly int _batchSize; @@ -70,8 +73,9 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable private readonly TimeSpan _deadLetterRetention; private readonly int _maxAttempts; private readonly Func _clock; + private readonly Func _drainGate; - private readonly SemaphoreSlim _drainGate = new(1, 1); + private readonly SemaphoreSlim _drainGateLock = new(1, 1); private Timer? _drainTimer; private TimeSpan _tickInterval; private volatile int _backoffIndex; @@ -85,13 +89,16 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable private string? _lastError; private HistorianDrainState _drainState = HistorianDrainState.Idle; private long _evictedCount; + // Tracks the last gate decision so a denial is logged on the transition rather than on every + // tick. Null until the first drain runs. + private bool? _lastGateDecision; private long _queuedRowCount; // Probe counter — incremented every time we actually issue a real COUNT(*) for // capacity enforcement. Public for test instrumentation only. private long _capacityProbeCount; // After every Nth enqueue we resync the in-memory counter from storage to defend - // against silent drift (e.g. an external process editing the DB). + // against silent drift (e.g. the peer replicating rows in underneath us). private const long ResyncEnqueueInterval = 10_000; private long _enqueuesSinceResync; @@ -99,9 +106,13 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable public long DebugCapacityProbeCount => Interlocked.Read(ref _capacityProbeCount); /// - /// Initializes a new instance of the class with the specified configuration. + /// Initializes a new instance of the class. /// - /// The filesystem path to the SQLite database file. + /// + /// The node's local database. Its alarm_sf_events table must already exist and be + /// registered for replication — the host does both in LocalDbSetup.OnReady, before + /// any consumer can resolve this sink. + /// /// The alarm historian writer to handle batch forwarding. /// The logger for diagnostic output. /// The maximum number of rows to forward in a single batch. Defaults to 100. @@ -109,18 +120,24 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable /// The timespan to retain dead-lettered rows before purging. Defaults to 30 days. /// The maximum number of delivery attempts before a perpetually-retrying (poison) row is dead-lettered. Defaults to 10. /// Optional clock function for testing; defaults to . - public SqliteStoreAndForwardSink( - string databasePath, + /// + /// Consulted at the top of every drain tick; false skips the tick entirely, leaving + /// the queue intact. Callers running a redundant pair MUST supply a gate that is true on at + /// most one node, because the queue replicates — see the class remarks. The default + /// (null ⇒ always drain) is the correct single-node and test posture. + /// + public LocalDbStoreAndForwardSink( + ILocalDb db, IAlarmHistorianWriter writer, ILogger logger, int batchSize = 100, long capacity = DefaultCapacity, TimeSpan? deadLetterRetention = null, int maxAttempts = DefaultMaxAttempts, - Func? clock = null) + Func? clock = null, + Func? drainGate = null) { - if (string.IsNullOrWhiteSpace(databasePath)) - throw new ArgumentException("Database path required.", nameof(databasePath)); + _db = db ?? throw new ArgumentNullException(nameof(db)); _writer = writer ?? throw new ArgumentNullException(nameof(writer)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _batchSize = batchSize > 0 ? batchSize : throw new ArgumentOutOfRangeException(nameof(batchSize)); @@ -128,50 +145,11 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable _deadLetterRetention = deadLetterRetention ?? DefaultDeadLetterRetention; _maxAttempts = maxAttempts > 0 ? maxAttempts : throw new ArgumentOutOfRangeException(nameof(maxAttempts)); _clock = clock ?? (() => DateTime.UtcNow); - // DefaultTimeout gives ADO.NET command-level retry; the PRAGMA busy_timeout - // applied in OpenConnection backs it with SQLite's own busy-handler so an - // enqueue/drain collision waits out the file lock instead of throwing - // SQLITE_BUSY immediately. - _connectionString = new SqliteConnectionStringBuilder - { - DataSource = databasePath, - DefaultTimeout = 5, - }.ToString(); + _drainGate = drainGate ?? (static () => true); - InitializeSchema(); _queuedRowCount = ProbeQueuedRowCount(); } - /// - /// Open a connection with the busy timeout + WAL journal applied. SQLite - /// serializes writers with a file lock; the busy_timeout lets a writer wait - /// out a competing lock (default is 0 — fail fast), and WAL lets readers and - /// the single writer proceed without blocking each other. - /// - private SqliteConnection OpenConnection() - { - var conn = new SqliteConnection(_connectionString); - conn.Open(); - ApplyPragmas(conn); - return conn; - } - - /// Apply busy_timeout + WAL pragmas to an already-open connection (sync). - private static void ApplyPragmas(SqliteConnection conn) - { - using var pragma = conn.CreateCommand(); - pragma.CommandText = "PRAGMA busy_timeout=5000; PRAGMA journal_mode=WAL;"; - pragma.ExecuteNonQuery(); - } - - /// Apply busy_timeout + WAL pragmas to an already-open connection (async). - private static async Task ApplyPragmasAsync(SqliteConnection conn, CancellationToken ct) - { - using var pragma = conn.CreateCommand(); - pragma.CommandText = "PRAGMA busy_timeout=5000; PRAGMA journal_mode=WAL;"; - await pragma.ExecuteNonQueryAsync(ct).ConfigureAwait(false); - } - /// /// Start the background drain worker. Not started automatically so tests can /// drive deterministically. @@ -188,7 +166,7 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable /// The base interval between drain attempts. public void StartDrainLoop(TimeSpan tickInterval) { - if (_disposed) throw new ObjectDisposedException(nameof(SqliteStoreAndForwardSink)); + if (_disposed) throw new ObjectDisposedException(nameof(LocalDbStoreAndForwardSink)); _tickInterval = tickInterval; _drainTimer?.Dispose(); // One-shot: dueTime = tickInterval, period = Infinite. RescheduleDrain re-arms @@ -235,29 +213,50 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable catch (ObjectDisposedException) { /* raced with Dispose — nothing to re-arm */ } } + /// + /// Derives the row's primary key from its payload. + /// + /// + /// Deterministic rather than a fresh GUID so that the same event enqueued independently on + /// both nodes of a pair converges to one row under last-writer-wins instead of duplicating. + /// That happens for real: HistorianAdapterActor default-writes while its redundancy + /// role is unknown, so in the window before the first snapshot arrives both adapters accept + /// the same fanned transition. Two distinct events cannot collide — + /// carries a full-precision timestamp alongside the alarm + /// id, kind, message and user, so an equal hash means an equal event. + /// + private static string DeriveId(string payloadJson) => + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(payloadJson))); + /// public async Task EnqueueAsync(AlarmHistorianEvent evt, CancellationToken cancellationToken) { if (evt is null) throw new ArgumentNullException(nameof(evt)); - if (_disposed) throw new ObjectDisposedException(nameof(SqliteStoreAndForwardSink)); + if (_disposed) throw new ObjectDisposedException(nameof(LocalDbStoreAndForwardSink)); - using var conn = new SqliteConnection(_connectionString); - await conn.OpenAsync(cancellationToken).ConfigureAwait(false); - await ApplyPragmasAsync(conn, cancellationToken).ConfigureAwait(false); + await EnforceCapacityFastPathAsync(cancellationToken).ConfigureAwait(false); - await EnforceCapacityFastPathAsync(conn, cancellationToken).ConfigureAwait(false); + var payload = JsonSerializer.Serialize(evt); - using var cmd = conn.CreateCommand(); - cmd.CommandText = """ - INSERT INTO Queue (AlarmId, EnqueuedUtc, PayloadJson, AttemptCount) - VALUES ($alarmId, $enqueued, $payload, 0); - """; - cmd.Parameters.AddWithValue("$alarmId", evt.AlarmId); - cmd.Parameters.AddWithValue("$enqueued", _clock().ToString("O")); - cmd.Parameters.AddWithValue("$payload", JsonSerializer.Serialize(evt)); - await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + // ON CONFLICT DO NOTHING: re-enqueuing an identical event (the pair's boot-window + // double-accept) must not resurrect a row the drain already bumped or dead-lettered. + var inserted = await _db.ExecuteAsync( + """ + INSERT INTO alarm_sf_events + (id, alarm_id, enqueued_at_utc, payload_json, attempt_count) + VALUES (@Id, @AlarmId, @EnqueuedAtUtc, @PayloadJson, 0) + ON CONFLICT(id) DO NOTHING + """, + new + { + Id = DeriveId(payload), + AlarmId = evt.AlarmId, + EnqueuedAtUtc = _clock().ToString("O"), + PayloadJson = payload, + }, + cancellationToken).ConfigureAwait(false); - Interlocked.Increment(ref _queuedRowCount); + if (inserted > 0) Interlocked.Increment(ref _queuedRowCount); } /// @@ -268,15 +267,17 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable /// through which still runs a precise /// COUNT to compute the exact number of rows to evict. /// - private async Task EnforceCapacityFastPathAsync(SqliteConnection conn, CancellationToken ct) + private async Task EnforceCapacityFastPathAsync(CancellationToken ct) { var enqueuesSinceResync = Interlocked.Increment(ref _enqueuesSinceResync); var cached = Interlocked.Read(ref _queuedRowCount); - // Periodic resync — bounded amount of drift even under exotic conditions. + // Periodic resync — bounded amount of drift even under exotic conditions. Now doubly + // warranted: replication applies the peer's rows straight into the table, so the counter + // has a second way to fall behind that no local code path can observe. if (enqueuesSinceResync >= ResyncEnqueueInterval) { - await ResyncQueuedRowCountAsync(conn, ct).ConfigureAwait(false); + await ResyncQueuedRowCountAsync(ct).ConfigureAwait(false); cached = Interlocked.Read(ref _queuedRowCount); Interlocked.Exchange(ref _enqueuesSinceResync, 0); } @@ -286,55 +287,70 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable // Cached counter says we're at or above the capacity wall — fall back to the // precise path which probes COUNT(*) and evicts whatever's needed. - await EnforceCapacityAsync(conn, ct).ConfigureAwait(false); + await EnforceCapacityAsync(ct).ConfigureAwait(false); } /// Synchronously query COUNT(*) of non-dead-lettered rows. Used at startup. private long ProbeQueuedRowCount() { Interlocked.Increment(ref _capacityProbeCount); - using var conn = OpenConnection(); + using var conn = _db.CreateConnection(); using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM Queue WHERE DeadLettered = 0"; + cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0"; return (long)(cmd.ExecuteScalar() ?? 0L); } /// Re-sync the in-memory counter from storage (async path). - private async Task ResyncQueuedRowCountAsync(SqliteConnection conn, CancellationToken ct) + private async Task ResyncQueuedRowCountAsync(CancellationToken ct) { Interlocked.Increment(ref _capacityProbeCount); - using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM Queue WHERE DeadLettered = 0"; - var live = (long)(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) ?? 0L); + var live = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false); Interlocked.Exchange(ref _queuedRowCount, live); } + private async Task CountAsync(string predicate, CancellationToken ct) + { + var rows = await _db.QueryAsync( + $"SELECT COUNT(*) FROM alarm_sf_events WHERE {predicate}", + r => r.GetInt64(0), + parameters: null, + ct).ConfigureAwait(false); + return rows.Count > 0 ? rows[0] : 0L; + } + /// /// Read up to queued rows, forward through the writer, /// remove Ack'd rows, dead-letter PermanentFail rows, and extend the backoff /// on RetryPlease. Safe to call from multiple threads; the semaphore enforces /// serial execution. /// + /// + /// Returns without touching the queue when the drain gate is closed. On a redundant pair + /// that is the Secondary's steady state: its table fills by replication and stays put until + /// it is promoted. + /// /// Cancellation token for the operation. /// A task that represents the asynchronous operation. public async Task DrainOnceAsync(CancellationToken ct) { if (_disposed) return; - if (!await _drainGate.WaitAsync(0, ct).ConfigureAwait(false)) return; + if (!await _drainGateLock.WaitAsync(0, ct).ConfigureAwait(false)) return; try { + if (!EvaluateDrainGate()) + { + lock (_statusLock) { _drainState = HistorianDrainState.NotPrimary; } + return; + } + lock (_statusLock) { _drainState = HistorianDrainState.Draining; _lastDrainUtc = _clock(); } - // One connection per drain tick — used by purge, read, corrupt-dead-letter, - // and the outcome-applying transaction. - using var conn = OpenConnection(); - - PurgeAgedDeadLetters(conn); - var batch = ReadBatch(conn); + await PurgeAgedDeadLettersAsync(ct).ConfigureAwait(false); + var batch = await ReadBatchAsync(ct).ConfigureAwait(false); if (batch.Count == 0) { lock (_statusLock) { _drainState = HistorianDrainState.Idle; } @@ -342,22 +358,24 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable } // A null/un-deserializable payload can never succeed — dead-letter it - // immediately for its own RowId so it cannot stall the queue head, and + // immediately for its own id so it cannot stall the queue head, and // exclude it from the batch handed to the writer. - var corruptRowIds = batch.Where(r => r.Event is null).Select(r => r.RowId).ToList(); + var corruptIds = batch.Where(r => r.Event is null).Select(r => r.Id).ToList(); var liveRows = batch.Where(r => r.Event is not null).ToList(); var events = liveRows.Select(r => r.Event!).ToList(); - if (corruptRowIds.Count > 0) + if (corruptIds.Count > 0) { - using var corruptTx = conn.BeginTransaction(); - foreach (var rowId in corruptRowIds) - DeadLetterRow(conn, corruptTx, rowId, $"corrupt payload at {_clock():O}"); - corruptTx.Commit(); - Interlocked.Add(ref _queuedRowCount, -corruptRowIds.Count); + await using (var corruptTx = await _db.BeginTransactionAsync(ct).ConfigureAwait(false)) + { + foreach (var id in corruptIds) + await DeadLetterRowAsync(corruptTx, id, $"corrupt payload at {_clock():O}", ct).ConfigureAwait(false); + await corruptTx.CommitAsync(ct).ConfigureAwait(false); + } + Interlocked.Add(ref _queuedRowCount, -corruptIds.Count); _logger.Warning( "Dead-lettered {Count} historian queue row(s) with un-deserializable payload", - corruptRowIds.Count); + corruptIds.Count); } if (events.Count == 0) @@ -404,41 +422,44 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable } int rowsLeavingQueue = 0; - using (var tx = conn.BeginTransaction()) + await using (var tx = await _db.BeginTransactionAsync(ct).ConfigureAwait(false)) { for (var i = 0; i < outcomes.Count; i++) { var outcome = outcomes[i]; - var rowId = liveRows[i].RowId; + var id = liveRows[i].Id; switch (outcome) { case HistorianWriteOutcome.Ack: - DeleteRow(conn, tx, rowId); + // Deleted, not marked delivered. The replication engine carries the + // delete as a tombstone, so the peer drops its copy and cannot + // re-deliver the row after a failover. + await DeleteRowAsync(tx, id, ct).ConfigureAwait(false); rowsLeavingQueue++; break; case HistorianWriteOutcome.PermanentFail: - DeadLetterRow(conn, tx, rowId, $"permanent fail at {_clock():O}"); + await DeadLetterRowAsync(tx, id, $"permanent fail at {_clock():O}", ct).ConfigureAwait(false); rowsLeavingQueue++; break; case HistorianWriteOutcome.RetryPlease: // finding 002: cap retries so a perpetually-RetryPlease (poison) // row cannot retry forever at the 60s backoff floor. The incoming - // AttemptCount is the count BEFORE this attempt; +1 accounts for the + // attempt count is the count BEFORE this attempt; +1 accounts for the // bump this drain represents. At the cap, dead-letter instead of // bumping — and count it as leaving the live queue like PermanentFail. if (liveRows[i].AttemptCount + 1 >= _maxAttempts) { - DeadLetterRow(conn, tx, rowId, $"max attempts ({_maxAttempts}) exceeded"); + await DeadLetterRowAsync(tx, id, $"max attempts ({_maxAttempts}) exceeded", ct).ConfigureAwait(false); rowsLeavingQueue++; } else { - BumpAttempt(conn, tx, rowId, "retry-please"); + await BumpAttemptAsync(tx, id, "retry-please", ct).ConfigureAwait(false); } break; } } - tx.Commit(); + await tx.CommitAsync(ct).ConfigureAwait(false); } // Ack-deleted + PermanentFail-dead-lettered rows both leave the // non-dead-lettered queue, as do RetryPlease rows that hit the max-attempts @@ -464,22 +485,66 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable } finally { - _drainGate.Release(); + _drainGateLock.Release(); } } + /// + /// Consults the injected gate, logging only when the decision changes. + /// + /// + /// Logged at all because the closed state is indistinguishable from a healthy idle queue + /// from the outside, and one way to reach it is a redundancy snapshot that never names this + /// node — the documented identity-mismatch shape. Without this line, that misconfiguration + /// would present as alarm history quietly ceasing on both nodes of a pair. Logged on the + /// transition rather than per tick because the drain runs every few seconds. + /// + /// true when this tick may proceed. + private bool EvaluateDrainGate() + { + bool open; + try + { + open = _drainGate(); + } + catch (Exception ex) + { + // A throwing gate must not wedge the queue silently, and must not be treated as + // permission either: skip this tick, say why, and re-ask on the next one. + _logger.Error(ex, "Historian drain gate threw; skipping this tick"); + return false; + } + + bool? previous; + lock (_statusLock) + { + previous = _lastGateDecision; + _lastGateDecision = open; + } + + if (previous == open) return open; + + if (open) + _logger.Information("Historian drain resumed — this node now services the alarm queue"); + else + _logger.Information( + "Historian drain suspended — this node is not the Primary. Queued alarm events are retained and will drain from whichever node holds the Primary role"); + + return open; + } + /// public HistorianSinkStatus GetStatus() { - if (_disposed) throw new ObjectDisposedException(nameof(SqliteStoreAndForwardSink)); + if (_disposed) throw new ObjectDisposedException(nameof(LocalDbStoreAndForwardSink)); var queued = Interlocked.Read(ref _queuedRowCount); if (queued < 0) queued = 0; long deadlettered; - using (var conn = OpenConnection()) + using (var conn = _db.CreateConnection()) using (var cmd = conn.CreateCommand()) { - cmd.CommandText = "SELECT COUNT(*) FROM Queue WHERE DeadLettered = 1"; + cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1"; deadlettered = (long)(cmd.ExecuteScalar() ?? 0L); } @@ -510,10 +575,13 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable /// The number of rows revived from the dead-letter state. public int RetryDeadLettered() { - if (_disposed) throw new ObjectDisposedException(nameof(SqliteStoreAndForwardSink)); - using var conn = OpenConnection(); + if (_disposed) throw new ObjectDisposedException(nameof(LocalDbStoreAndForwardSink)); + // Through a LocalDb connection, not a private one: the capture triggers are what make this + // revival replicate to the peer, and they only exist on the registered table. + using var conn = _db.CreateConnection(); using var cmd = conn.CreateCommand(); - cmd.CommandText = "UPDATE Queue SET DeadLettered = 0, AttemptCount = 0, LastError = NULL WHERE DeadLettered = 1"; + cmd.CommandText = + "UPDATE alarm_sf_events SET dead_lettered = 0, attempt_count = 0, last_error = NULL WHERE dead_lettered = 1"; var revived = cmd.ExecuteNonQuery(); // Dead-lettered rows rejoin the non-dead-lettered queue — keep the in-memory // counter aligned. @@ -523,29 +591,31 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable /// /// One queued row paired with its deserialized event. is - /// null when the row's PayloadJson is corrupt or un-deserializable — - /// the always stays bound to its own row so outcomes can + /// null when the row's payload_json is corrupt or un-deserializable — + /// the always stays bound to its own row so outcomes can /// never be mapped to the wrong row. /// - private readonly record struct QueueRow(long RowId, AlarmHistorianEvent? Event, long AttemptCount); + private readonly record struct QueueRow(string Id, AlarmHistorianEvent? Event, long AttemptCount); - private List ReadBatch(SqliteConnection conn) + private async Task> ReadBatchAsync(CancellationToken ct) { - var rows = new List(); - using var cmd = conn.CreateCommand(); - cmd.CommandText = """ - SELECT RowId, PayloadJson, AttemptCount FROM Queue - WHERE DeadLettered = 0 - ORDER BY RowId ASC - LIMIT $limit - """; - cmd.Parameters.AddWithValue("$limit", _batchSize); - using var reader = cmd.ExecuteReader(); - while (reader.Read()) + // Ordered by enqueue time rather than insertion order: a hashed TEXT primary key carries + // no sequence the way the legacy AUTOINCREMENT rowid did. Round-trip ("O") timestamps sort + // lexicographically in chronological order; id makes the ordering total. + var raw = await _db.QueryAsync( + """ + SELECT id, payload_json, attempt_count FROM alarm_sf_events + WHERE dead_lettered = 0 + ORDER BY enqueued_at_utc ASC, id ASC + LIMIT @Limit + """, + r => (Id: r.GetString(0), Payload: r.GetString(1), AttemptCount: r.GetInt64(2)), + new { Limit = _batchSize }, + ct).ConfigureAwait(false); + + var rows = new List(raw.Count); + foreach (var (id, payload, attemptCount) in raw) { - var rowId = reader.GetInt64(0); - var payload = reader.GetString(1); - var attemptCount = reader.GetInt64(2); AlarmHistorianEvent? evt; try { @@ -556,77 +626,58 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable // Malformed JSON — carry a null event so the caller dead-letters this row. evt = null; } - rows.Add(new QueueRow(rowId, evt, attemptCount)); + rows.Add(new QueueRow(id, evt, attemptCount)); } return rows; } - private static void DeleteRow(SqliteConnection conn, SqliteTransaction tx, long rowId) - { - using var cmd = conn.CreateCommand(); - cmd.Transaction = tx; - cmd.CommandText = "DELETE FROM Queue WHERE RowId = $id"; - cmd.Parameters.AddWithValue("$id", rowId); - cmd.ExecuteNonQuery(); - } + private static Task DeleteRowAsync(ILocalDbTransaction tx, string id, CancellationToken ct) => + tx.ExecuteAsync("DELETE FROM alarm_sf_events WHERE id = @Id", new { Id = id }, ct); - private void DeadLetterRow(SqliteConnection conn, SqliteTransaction tx, long rowId, string reason) - { - using var cmd = conn.CreateCommand(); - cmd.Transaction = tx; - cmd.CommandText = """ - UPDATE Queue SET DeadLettered = 1, LastAttemptUtc = $now, LastError = $err, AttemptCount = AttemptCount + 1 - WHERE RowId = $id - """; - cmd.Parameters.AddWithValue("$now", _clock().ToString("O")); - cmd.Parameters.AddWithValue("$err", reason); - cmd.Parameters.AddWithValue("$id", rowId); - cmd.ExecuteNonQuery(); - } + private Task DeadLetterRowAsync(ILocalDbTransaction tx, string id, string reason, CancellationToken ct) => + tx.ExecuteAsync( + """ + UPDATE alarm_sf_events + SET dead_lettered = 1, last_attempt_utc = @Now, last_error = @Err, + attempt_count = attempt_count + 1 + WHERE id = @Id + """, + new { Now = _clock().ToString("O"), Err = reason, Id = id }, + ct); - private void BumpAttempt(SqliteConnection conn, SqliteTransaction tx, long rowId, string reason) - { - using var cmd = conn.CreateCommand(); - cmd.Transaction = tx; - cmd.CommandText = """ - UPDATE Queue SET LastAttemptUtc = $now, LastError = $err, AttemptCount = AttemptCount + 1 - WHERE RowId = $id - """; - cmd.Parameters.AddWithValue("$now", _clock().ToString("O")); - cmd.Parameters.AddWithValue("$err", reason); - cmd.Parameters.AddWithValue("$id", rowId); - cmd.ExecuteNonQuery(); - } + private Task BumpAttemptAsync(ILocalDbTransaction tx, string id, string reason, CancellationToken ct) => + tx.ExecuteAsync( + """ + UPDATE alarm_sf_events + SET last_attempt_utc = @Now, last_error = @Err, attempt_count = attempt_count + 1 + WHERE id = @Id + """, + new { Now = _clock().ToString("O"), Err = reason, Id = id }, + ct); - // Async variant used by EnqueueAsync. - private async Task EnforceCapacityAsync(SqliteConnection conn, CancellationToken ct) + private async Task EnforceCapacityAsync(CancellationToken ct) { Interlocked.Increment(ref _capacityProbeCount); - long count; - using (var cmd = conn.CreateCommand()) - { - cmd.CommandText = "SELECT COUNT(*) FROM Queue WHERE DeadLettered = 0"; - count = (long)(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) ?? 0L); - } + var count = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false); + // Resync the in-memory counter while we have a fresh number. Interlocked.Exchange(ref _queuedRowCount, count); if (count < _capacity) return; var toEvict = count - _capacity + 1; - using (var cmd = conn.CreateCommand()) - { - cmd.CommandText = """ - DELETE FROM Queue - WHERE RowId IN ( - SELECT RowId FROM Queue - WHERE DeadLettered = 0 - ORDER BY RowId ASC - LIMIT $n - ) - """; - cmd.Parameters.AddWithValue("$n", toEvict); - await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); - } + await _db.ExecuteAsync( + """ + DELETE FROM alarm_sf_events + WHERE id IN ( + SELECT id FROM alarm_sf_events + WHERE dead_lettered = 0 + ORDER BY enqueued_at_utc ASC, id ASC + LIMIT @N + ) + """, + new { N = toEvict }, + ct).ConfigureAwait(false); + Interlocked.Add(ref _queuedRowCount, -toEvict); long lifetimeEvicted; lock (_statusLock) { _evictedCount += toEvict; lifetimeEvicted = _evictedCount; } @@ -635,40 +686,21 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable _capacity, toEvict, lifetimeEvicted); } - private void PurgeAgedDeadLetters(SqliteConnection conn) + private async Task PurgeAgedDeadLettersAsync(CancellationToken ct) { var cutoff = (_clock() - _deadLetterRetention).ToString("O"); - using var cmd = conn.CreateCommand(); - cmd.CommandText = """ - DELETE FROM Queue - WHERE DeadLettered = 1 AND LastAttemptUtc IS NOT NULL AND LastAttemptUtc < $cutoff - """; - cmd.Parameters.AddWithValue("$cutoff", cutoff); - var purged = cmd.ExecuteNonQuery(); + var purged = await _db.ExecuteAsync( + """ + DELETE FROM alarm_sf_events + WHERE dead_lettered = 1 AND last_attempt_utc IS NOT NULL AND last_attempt_utc < @Cutoff + """, + new { Cutoff = cutoff }, + ct).ConfigureAwait(false); + if (purged > 0) _logger.Information("Purged {Count} dead-lettered row(s) past retention window", purged); } - private void InitializeSchema() - { - using var conn = OpenConnection(); - using var cmd = conn.CreateCommand(); - cmd.CommandText = """ - CREATE TABLE IF NOT EXISTS Queue ( - RowId INTEGER PRIMARY KEY AUTOINCREMENT, - AlarmId TEXT NOT NULL, - EnqueuedUtc TEXT NOT NULL, - PayloadJson TEXT NOT NULL, - AttemptCount INTEGER NOT NULL DEFAULT 0, - LastAttemptUtc TEXT NULL, - LastError TEXT NULL, - DeadLettered INTEGER NOT NULL DEFAULT 0 - ); - CREATE INDEX IF NOT EXISTS IX_Queue_Drain ON Queue (DeadLettered, RowId); - """; - cmd.ExecuteNonQuery(); - } - private void BumpBackoff() => _backoffIndex = Math.Min(_backoffIndex + 1, BackoffLadder.Length - 1); private void ResetBackoff() => _backoffIndex = 0; @@ -676,12 +708,16 @@ public sealed class SqliteStoreAndForwardSink : IAlarmHistorianSink, IDisposable public TimeSpan CurrentBackoff => BackoffLadder[_backoffIndex]; /// Disposes the sink and releases all held resources including the drain timer and the writer. + /// + /// The is NOT disposed here — it is a shared singleton owned by the + /// host and used by the deployment cache too. + /// public void Dispose() { if (_disposed) return; _disposed = true; _drainTimer?.Dispose(); - _drainGate.Dispose(); + _drainGateLock.Dispose(); if (_writer is IDisposable writerDisposable) writerDisposable.Dispose(); } } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj index 62515df4..7e225b8d 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj @@ -20,6 +20,12 @@ --> + + diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/appsettings.json b/src/Server/ZB.MOM.WW.OtOpcUa.Host/appsettings.json index e7ca5cd6..68dce0ba 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/appsettings.json +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/appsettings.json @@ -52,9 +52,8 @@ "MaxBackoffSeconds": 30 }, "AlarmHistorian": { - "_comment": "Durable SQLite store-and-forward alarm sink. Drains alarm events to the ServerHistorian gateway's SendEvent path; the downstream connection (endpoint/key/TLS) is sourced from the ServerHistorian section.", + "_comment": "Durable store-and-forward alarm sink. Buffers into the node's consolidated LocalDb (see the LocalDb section) so the queue replicates to the redundant pair peer, and drains alarm events to the ServerHistorian gateway's SendEvent path from whichever node holds the Primary role. The downstream connection (endpoint/key/TLS) is sourced from the ServerHistorian section.", "Enabled": false, - "DatabasePath": "alarm-historian.db", "DrainIntervalSeconds": 5, "Capacity": 1000000, "DeadLetterRetentionDays": 30 diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index cc0c598f..b8dbd478 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -24,6 +24,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Core.VirtualTags; using ZB.MOM.WW.OtOpcUa.OpcUaServer; using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache; +using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; using CommonsNodeId = ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId; @@ -252,6 +253,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers /// non-cluster ActorRefProvider). See . private readonly Func? _driverMemberCountProvider; + /// Where the Primary-gate decision is published for consumers that live outside a mailbox — + /// today, the alarm store-and-forward drain. Null on nodes that do not wire one. + private readonly IRedundancyRoleView? _redundancyRoleView; + /// Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent /// identity mismatch (03/S5) would otherwise log on every snapshot. private bool _warnedSnapshotMissingLocalNode; @@ -366,7 +371,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers IActorRef? scriptedAlarmHostOverride = null, IDriverCapabilityInvokerFactory? invokerFactory = null, Func? driverMemberCountProvider = null, - IDeploymentArtifactCache? deploymentArtifactCache = null) => + IDeploymentArtifactCache? deploymentArtifactCache = null, + IRedundancyRoleView? redundancyRoleView = null) => // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an // expression tree. Six IActorRef? parameters and several interface-typed ones mean a // mis-ordered argument is usually type-compatible and therefore compiles clean, then binds @@ -376,7 +382,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor, healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride, loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider, - deploymentArtifactCache)); + deploymentArtifactCache, redundancyRoleView)); /// Initializes a new DriverHostActor with the specified dependencies. /// Database context factory for configuration database access. @@ -426,9 +432,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers IActorRef? scriptedAlarmHostOverride = null, IDriverCapabilityInvokerFactory? invokerFactory = null, Func? driverMemberCountProvider = null, - IDeploymentArtifactCache? deploymentArtifactCache = null) + IDeploymentArtifactCache? deploymentArtifactCache = null, + IRedundancyRoleView? redundancyRoleView = null) { _deploymentArtifactCache = deploymentArtifactCache; + _redundancyRoleView = redundancyRoleView; _dbFactory = dbFactory; _localNode = localNode; _coordinatorOverride = coordinator; @@ -1476,19 +1484,23 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers if (local is not null) { _localRole = local.Role; - return; } - - // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the - // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent - // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable. - if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1) + else if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1) { + // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the + // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent + // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable. _warnedSnapshotMissingLocalNode = true; _log.Warning( "DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster", _localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId))); } + + // Publish on EVERY snapshot, including ones that leave the cached role untouched: the other + // half of the decision is the driver member count, which moves with cluster membership and + // not with this message. Snapshots are re-published on a heartbeat, so a membership change + // is picked up within one heartbeat rather than waiting for a role change that may never come. + _redundancyRoleView?.Publish(ShouldServiceAsPrimary()); } private void Stale() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/AlarmHistorianOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/AlarmHistorianOptions.cs index 6bced110..9d7ed4a7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/AlarmHistorianOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/AlarmHistorianOptions.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using System.IO; using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian; @@ -7,11 +6,12 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian; /// /// Binds the AlarmHistorian configuration section that gates the durable /// store-and-forward alarm sink. When is true, -/// AddAlarmHistorian registers a SqliteStoreAndForwardSink (draining to the +/// AddAlarmHistorian registers a LocalDbStoreAndForwardSink (draining to the /// gateway alarm writer supplied by the Host) in place of the /// NullAlarmHistorianSink default; otherwise the Null default survives. This section -/// supplies only the gate and the SQLite store-and-forward knobs — the -/// downstream connection (endpoint/key/TLS) is sourced from the ServerHistorian section. +/// supplies only the gate and the store-and-forward knobs — the +/// downstream connection (endpoint/key/TLS) is sourced from the ServerHistorian section, +/// and the queue's storage location is the node's consolidated LocalDb:Path database. /// public sealed class AlarmHistorianOptions { @@ -24,9 +24,6 @@ public sealed class AlarmHistorianOptions /// public bool Enabled { get; init; } - /// Filesystem path to the local SQLite store-and-forward queue database. - public string DatabasePath { get; init; } = "alarm-historian.db"; - /// Maximum number of queued rows the drain worker forwards in a single batch. public int BatchSize { get; init; } = 100; @@ -34,15 +31,15 @@ public sealed class AlarmHistorianOptions public int DrainIntervalSeconds { get; init; } = 5; /// Maximum queued rows before the sink evicts the oldest. Defaults to 1,000,000 - /// (matches SqliteStoreAndForwardSink's DefaultCapacity). - public long Capacity { get; init; } = SqliteStoreAndForwardSink.DefaultCapacity; + /// (matches LocalDbStoreAndForwardSink's DefaultCapacity). + public long Capacity { get; init; } = LocalDbStoreAndForwardSink.DefaultCapacity; /// Days to retain dead-lettered rows before purge. Defaults to 30. public int DeadLetterRetentionDays { get; init; } = 30; /// Maximum delivery attempts before a perpetually-retrying (poison) row is dead-lettered. - /// Defaults to 10 (matches SqliteStoreAndForwardSink's DefaultMaxAttempts). - public int MaxAttempts { get; init; } = SqliteStoreAndForwardSink.DefaultMaxAttempts; + /// Defaults to 10 (matches LocalDbStoreAndForwardSink's DefaultMaxAttempts). + public int MaxAttempts { get; init; } = LocalDbStoreAndForwardSink.DefaultMaxAttempts; /// Returns operator-facing misconfiguration warnings for an Enabled historian /// (empty when disabled or correctly configured). Pure — the registration logs each entry. @@ -51,8 +48,6 @@ public sealed class AlarmHistorianOptions { var warnings = new List(); if (!Enabled) return warnings; - if (!Path.IsPathRooted(DatabasePath)) - warnings.Add($"AlarmHistorian:DatabasePath '{DatabasePath}' is relative — it resolves against the process working directory (e.g. System32 for a Windows service). Set an absolute path."); if (DrainIntervalSeconds <= 0) warnings.Add($"AlarmHistorian:DrainIntervalSeconds is {DrainIntervalSeconds} — must be > 0; the drain timer will throw or spin at startup."); if (Capacity <= 0) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs new file mode 100644 index 00000000..a74797f5 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs @@ -0,0 +1,46 @@ +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; + +/// +/// A singleton snapshot of the Primary-gate decision, so code that lives outside an actor can +/// ask "should this node be servicing Primary-only work right now?". +/// +/// +/// +/// Exists for the alarm store-and-forward drain. That drain runs on a timer owned by the +/// sink, not on an actor mailbox, so it cannot receive RedundancyStateChanged +/// directly — but it must be Primary-scoped, because the queue it drains replicates to the +/// pair peer and two draining nodes would deliver every event twice. +/// +/// +/// Deliberately publishes the decision rather than the role and member count that +/// produced it, so stays the single place that decides. +/// +/// +public interface IRedundancyRoleView +{ + /// Whether this node should service Primary-only work right now. + bool ShouldServiceAsPrimary { get; } + + /// Records a fresh decision. Called by DriverHostActor on every redundancy snapshot. + /// The decision produced. + void Publish(bool shouldServiceAsPrimary); +} + +/// Thread-safe backed by a volatile field. +public sealed class RedundancyRoleView : IRedundancyRoleView +{ + // Seeded with the decision for "role unknown, no driver peer" rather than a bare false. An + // unpublished view and a node with no peer are genuinely the same situation, and they must + // behave the same: a deployment that runs no redundancy at all never publishes here, and + // defaulting closed would silently stop its alarm history forever. + private volatile bool _shouldServiceAsPrimary = + PrimaryGatePolicy.ShouldServiceAsPrimary(localRole: null, driverMemberCount: 0); + + /// + public bool ShouldServiceAsPrimary => _shouldServiceAsPrimary; + + /// + public void Publish(bool shouldServiceAsPrimary) => _shouldServiceAsPrimary = shouldServiceAsPrimary; +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs index 0cddd870..7dc1e921 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs @@ -20,7 +20,9 @@ using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.Health; using ZB.MOM.WW.OtOpcUa.Runtime.Historian; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; +using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; +using ZB.MOM.WW.LocalDb; namespace ZB.MOM.WW.OtOpcUa.Runtime; @@ -65,12 +67,23 @@ public static class ServiceCollectionExtensions /// /// Config-gated durable alarm-historian sink. When the AlarmHistorian section has - /// Enabled=true, registers a (draining via the + /// Enabled=true, registers a (draining via the /// -supplied writer) as the , /// overriding the default. Otherwise a no-op (Null stays). /// The writer is injected so the durable downstream (the HistorianGateway alarm writer) can be /// supplied by the Host, which is the only project that references it. /// + /// + /// + /// The queue lives in the node's consolidated , which replicates it + /// to the redundant pair peer — so undelivered alarm history survives losing the node + /// holding it. That is also why the drain is gated on : + /// the Secondary holds a full replica of the Primary's queue, and an ungated drain there + /// would re-deliver every event. Both services are resolved from the provider, so a + /// deployment that enables this section without registering LocalDb fails loudly at + /// resolution rather than silently buffering to nowhere. + /// + /// /// The service collection to register with. /// The configuration carrying the AlarmHistorian section. /// @@ -87,21 +100,28 @@ public static class ServiceCollectionExtensions if (opts is not { Enabled: true }) return services; // leave the Null default from AddOtOpcUaRuntime foreach (var warning in opts.Validate()) - Serilog.Log.Logger.ForContext().Warning("Historian config: {HistorianConfigWarning}", warning); + Serilog.Log.Logger.ForContext().Warning("Historian config: {HistorianConfigWarning}", warning); + + // The view is a plain singleton so it exists even on nodes whose DriverHostActor never + // publishes to it — an unpublished view reads as "no peer, drain", which is the correct + // posture for a deployment that runs no redundancy at all. + services.TryAddSingleton(); services.AddSingleton(sp => { - // SqliteStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging). + // LocalDbStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging). // Resolve it off the host's configured static logger so the drain worker's WARN/INFO // lines land in the same sinks as the rest of the process. - var sink = new SqliteStoreAndForwardSink( - opts.DatabasePath, + var roleView = sp.GetRequiredService(); + var sink = new LocalDbStoreAndForwardSink( + sp.GetRequiredService(), writerFactory(opts, sp), - Serilog.Log.Logger.ForContext(), + Serilog.Log.Logger.ForContext(), batchSize: opts.BatchSize, capacity: opts.Capacity, deadLetterRetention: TimeSpan.FromDays(opts.DeadLetterRetentionDays), - maxAttempts: opts.MaxAttempts); + maxAttempts: opts.MaxAttempts, + drainGate: () => roleView.ShouldServiceAsPrimary); sink.StartDrainLoop(TimeSpan.FromSeconds(opts.DrainIntervalSeconds)); return sink; }); @@ -227,6 +247,11 @@ public static class ServiceCollectionExtensions // harnesses) rather than given a null-object, so DriverHostActor skips caching outright // instead of pretending to cache into a sink that drops everything. var deploymentArtifactCache = resolver.GetService(); + // Where this actor publishes its Primary-gate verdict for the alarm store-and-forward + // drain, which runs on a timer and so cannot read RedundancyStateChanged itself. + // Registered by AddAlarmHistorian; absent when no durable sink is configured, in which + // case there is nothing downstream to inform. + var redundancyRoleView = resolver.GetService(); // Root script logger backs the ScriptedAlarm host's engine + script logging. Registered in // Host DI inside the hasDriver block; may be absent in some role configs / test harnesses, // in which case the DriverHostActor gracefully skips spawning the ScriptedAlarm host. @@ -345,7 +370,8 @@ public static class ServiceCollectionExtensions loggerFactory: loggerFactory, scriptRootLogger: scriptRootLogger, invokerFactory: invokerFactory, - deploymentArtifactCache: deploymentArtifactCache), + deploymentArtifactCache: deploymentArtifactCache, + redundancyRoleView: redundancyRoleView), DriverHostActorName); registry.Register(driverHost); diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/SqliteStoreAndForwardSinkTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs similarity index 77% rename from tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/SqliteStoreAndForwardSinkTests.cs rename to tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs index 3f86e70f..da64ddf4 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/SqliteStoreAndForwardSinkTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs @@ -1,9 +1,12 @@ using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Serilog; using Serilog.Core; using Serilog.Events; using Shouldly; using Xunit; +using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests; @@ -14,22 +17,66 @@ namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests; /// capacity eviction, and retention-based dead-letter purge. /// [Trait("Category", "Unit")] -public sealed class SqliteStoreAndForwardSinkTests : IDisposable +public sealed class LocalDbStoreAndForwardSinkTests : IDisposable { private readonly string _dbPath; private readonly ILogger _log; + private ServiceProvider? _provider; + private ILocalDb? _db; /// Initializes a new test instance. - public SqliteStoreAndForwardSinkTests() + public LocalDbStoreAndForwardSinkTests() { _dbPath = Path.Combine(Path.GetTempPath(), $"otopcua-historian-{Guid.NewGuid():N}.sqlite"); _log = new LoggerConfiguration().MinimumLevel.Verbose().CreateLogger(); } + /// + /// A real temp-file local database with the buffer table created and registered, built on + /// first use. + /// + /// + /// A real database rather than a stub. The library has no in-memory mode, and a bare + /// would lack the zb_hlc_next() UDF the capture + /// triggers call — so every write to a registered table would fail closed. Registering here + /// also means these tests exercise the same capture path production replicates through. + /// + private ILocalDb Db + { + get + { + if (_db is not null) return _db; + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["LocalDb:Path"] = _dbPath }) + .Build(); + + _provider = new ServiceCollection() + .AddZbLocalDb(configuration, db => + { + using var connection = db.CreateConnection(); + AlarmSfSchema.Apply(connection); + db.RegisterReplicated(AlarmSfSchema.EventsTable); + }) + .BuildServiceProvider(); + + return _db = _provider.GetRequiredService(); + } + } + /// Cleans up test resources. public void Dispose() { - try { if (File.Exists(_dbPath)) File.Delete(_dbPath); } catch { } + _provider?.Dispose(); + + // Pooled connections keep a handle open past dispose; clearing the pools first is what + // makes the delete actually succeed. + SqliteConnection.ClearAllPools(); + + foreach (var path in new[] { _dbPath, $"{_dbPath}-wal", $"{_dbPath}-shm" }) + { + try { if (File.Exists(path)) File.Delete(path); } catch { /* best effort */ } + } } private sealed class FakeWriter : IAlarmHistorianWriter @@ -81,7 +128,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task EnqueueThenDrain_Ack_removes_row() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); sink.GetStatus().QueueDepth.ShouldBe(1); @@ -102,7 +149,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Drain_with_empty_queue_is_noop() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.DrainOnceAsync(CancellationToken.None); @@ -116,7 +163,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable { var writer = new FakeWriter(); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.RetryPlease); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); var before = sink.CurrentBackoff; @@ -133,7 +180,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable { var writer = new FakeWriter(); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.RetryPlease); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); await sink.DrainOnceAsync(CancellationToken.None); @@ -153,7 +200,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable var writer = new FakeWriter(); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.PermanentFail); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.Ack); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("bad"), CancellationToken.None); await sink.EnqueueAsync(Event("good"), CancellationToken.None); @@ -169,7 +216,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Writer_exception_treated_as_retry_for_whole_batch() { var writer = new FakeWriter { ThrowOnce = new InvalidOperationException("pipe broken") }; - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); await sink.DrainOnceAsync(CancellationToken.None); @@ -189,8 +236,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Capacity_eviction_drops_oldest_nondeadlettered_row() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, batchSize: 100, capacity: 3); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, batchSize: 100, capacity: 3); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); await sink.EnqueueAsync(Event("A2"), CancellationToken.None); @@ -217,8 +264,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable var writer = new FakeWriter(); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.PermanentFail); - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, deadLetterRetention: TimeSpan.FromDays(30), + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, deadLetterRetention: TimeSpan.FromDays(30), clock: () => clock); await sink.EnqueueAsync(Event("bad"), CancellationToken.None); @@ -238,7 +285,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable { var writer = new FakeWriter(); writer.NextOutcomePerEvent.Enqueue(HistorianWriteOutcome.PermanentFail); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("bad"), CancellationToken.None); await sink.DrainOnceAsync(CancellationToken.None); @@ -257,7 +304,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Backoff_ladder_caps_at_60s() { var writer = new FakeWriter { DefaultOutcome = HistorianWriteOutcome.RetryPlease }; - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); @@ -278,8 +325,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task RetryPlease_dead_letters_row_after_MaxAttempts() { var writer = new FakeWriter { DefaultOutcome = HistorianWriteOutcome.RetryPlease }; - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, maxAttempts: 3); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, maxAttempts: 3); await sink.EnqueueAsync(Event("poison"), CancellationToken.None); @@ -316,17 +363,89 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable await NullAlarmHistorianSink.Instance.EnqueueAsync(Event("A1"), CancellationToken.None); } + /// + /// A closed drain gate must leave the queue completely alone — no writer call, no attempt + /// bump, no purge. + /// + /// + /// This is the property that keeps alarm delivery exactly-once across a redundant pair. + /// The buffer replicates, so the Secondary holds a full copy of the Primary's queue; a + /// Secondary that drained would re-send every event, continuously. + /// + [Fact] + public async Task Drain_does_nothing_while_the_gate_is_closed() + { + var writer = new FakeWriter(); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log, drainGate: () => false); + + await sink.EnqueueAsync(Event("A1"), CancellationToken.None); + await sink.EnqueueAsync(Event("A2"), CancellationToken.None); + + await sink.DrainOnceAsync(CancellationToken.None); + + writer.Batches.Count.ShouldBe(0, "a gated-off node must not deliver"); + + var status = sink.GetStatus(); + status.QueueDepth.ShouldBe(2, "the rows stay queued for whichever node holds the Primary role"); + status.DrainState.ShouldBe( + HistorianDrainState.NotPrimary, + "a Secondary's rising queue must be distinguishable from a stalled drain"); + } + + /// + /// The positive control for the case above: the same queue, the same sink, drains as soon + /// as the gate opens. Without this, a sink that was simply broken would pass the + /// gated-off assertion. + /// + [Fact] + public async Task Drain_delivers_the_retained_queue_once_the_gate_opens() + { + var writer = new FakeWriter(); + var isPrimary = false; + // ReSharper disable once AccessToModifiedClosure — flipping it mid-test is the point. + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log, drainGate: () => isPrimary); + + await sink.EnqueueAsync(Event("A1"), CancellationToken.None); + await sink.DrainOnceAsync(CancellationToken.None); + writer.Batches.Count.ShouldBe(0); + + isPrimary = true; // this node is promoted + await sink.DrainOnceAsync(CancellationToken.None); + + writer.Batches.Count.ShouldBe(1); + writer.Batches[0].Select(e => e.AlarmId).ShouldBe(["A1"]); + sink.GetStatus().QueueDepth.ShouldBe(0); + } + + /// + /// A gate that throws must be read as "not now", never as permission. Draining on a gate + /// fault would put a pair into dual-delivery precisely when its redundancy signal is sick. + /// + [Fact] + public async Task Drain_treats_a_throwing_gate_as_closed() + { + var writer = new FakeWriter(); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, drainGate: () => throw new InvalidOperationException("role unavailable")); + + await sink.EnqueueAsync(Event("A1"), CancellationToken.None); + await sink.DrainOnceAsync(CancellationToken.None); + + writer.Batches.Count.ShouldBe(0, "a faulted gate must fail closed"); + sink.GetStatus().QueueDepth.ShouldBe(1); + } + /// Verifies that the constructor rejects invalid arguments. [Fact] public void Ctor_rejects_bad_args() { var w = new FakeWriter(); - Should.Throw(() => new SqliteStoreAndForwardSink("", w, _log)); - Should.Throw(() => new SqliteStoreAndForwardSink(_dbPath, null!, _log)); - Should.Throw(() => new SqliteStoreAndForwardSink(_dbPath, w, null!)); - Should.Throw(() => new SqliteStoreAndForwardSink(_dbPath, w, _log, batchSize: 0)); - Should.Throw(() => new SqliteStoreAndForwardSink(_dbPath, w, _log, capacity: 0)); - Should.Throw(() => new SqliteStoreAndForwardSink(_dbPath, w, _log, maxAttempts: 0)); + Should.Throw(() => new LocalDbStoreAndForwardSink(null!, w, _log)); + Should.Throw(() => new LocalDbStoreAndForwardSink(Db, null!, _log)); + Should.Throw(() => new LocalDbStoreAndForwardSink(Db, w, null!)); + Should.Throw(() => new LocalDbStoreAndForwardSink(Db, w, _log, batchSize: 0)); + Should.Throw(() => new LocalDbStoreAndForwardSink(Db, w, _log, capacity: 0)); + Should.Throw(() => new LocalDbStoreAndForwardSink(Db, w, _log, maxAttempts: 0)); } /// Verifies that a disposed sink rejects enqueue operations. @@ -334,7 +453,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Disposed_sink_rejects_enqueue() { var writer = new FakeWriter(); - var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); sink.Dispose(); await Should.ThrowAsync( @@ -352,11 +471,14 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Drain_with_corrupt_payload_row_deadletters_it_and_keeps_good_rows_aligned() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + var clock = new TickingClock(); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log, clock: clock.Next); - // Row 1: good. Row 2: corrupt JSON (inserted directly). Row 3: good. + // Row 1: good. Row 2: corrupt JSON (inserted directly). Row 3: good. The shared ticking + // clock is what pins the corrupt row BETWEEN the good ones — which is the whole point of + // this case, as distinct from the corrupt-at-the-head case below. await sink.EnqueueAsync(Event("good-1"), CancellationToken.None); - InsertCorruptRow("corrupt"); + InsertCorruptRow("corrupt", clock.Next()); await sink.EnqueueAsync(Event("good-2"), CancellationToken.None); await sink.DrainOnceAsync(CancellationToken.None); @@ -383,9 +505,10 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Drain_with_corrupt_head_row_does_not_stall_queue() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + var clock = new TickingClock(); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log, clock: clock.Next); - InsertCorruptRow("corrupt-head"); + InsertCorruptRow("corrupt-head", clock.Next()); await sink.EnqueueAsync(Event("good-1"), CancellationToken.None); await sink.DrainOnceAsync(CancellationToken.None); @@ -409,7 +532,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task StartDrainLoop_honors_backoff_and_slows_cadence_under_retry() { var writer = new FakeWriter { DefaultOutcome = HistorianWriteOutcome.RetryPlease }; - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); @@ -436,7 +559,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task StartDrainLoop_keeps_steady_cadence_when_writer_is_healthy() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); sink.StartDrainLoop(TimeSpan.FromMilliseconds(30)); @@ -465,7 +588,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable // exception... so to exercise the *callback* catch we instead make the // fault originate from the writer itself but assert the loop self-heals. var writer = new ThrowingThenHealingWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); sink.StartDrainLoop(TimeSpan.FromMilliseconds(30)); @@ -490,7 +613,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Concurrent_enqueue_and_drain_do_not_throw_sqlite_busy() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); var faults = new List(); var enqueuers = Enumerable.Range(0, 4).Select(t => Task.Run(async () => @@ -561,7 +684,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Writer_returning_wrong_cardinality_outcomes_sets_backing_off_and_keeps_rows() { var writer = new WrongCardinalityWriter(returnExtraOutcome: true); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); await sink.EnqueueAsync(Event("A2"), CancellationToken.None); @@ -591,8 +714,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Capacity_eviction_increments_evicted_count_on_status() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, batchSize: 100, capacity: 3); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, batchSize: 100, capacity: 3); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); await sink.EnqueueAsync(Event("A2"), CancellationToken.None); @@ -616,7 +739,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task GetStatus_snapshot_is_consistent_under_concurrent_drain() { var writer = new FakeWriter { DefaultOutcome = HistorianWriteOutcome.RetryPlease }; - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); // Fill the queue. for (var i = 0; i < 10; i++) @@ -690,8 +813,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task EnqueueAsync_does_not_count_all_rows_on_every_call_below_capacity() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, batchSize: 100, capacity: 10_000); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, batchSize: 100, capacity: 10_000); for (var i = 0; i < 200; i++) await sink.EnqueueAsync(Event($"E{i}"), CancellationToken.None); @@ -705,7 +828,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable /// /// Regression for Core.AlarmHistorian-008: across every queue mutation (enqueue, /// Ack drain, PermanentFail drain, capacity eviction, RetryDeadLettered) the - /// queue depth reported by must + /// queue depth reported by must /// stay aligned with a fresh COUNT(*) against the database. Catches drift /// bugs in the in-memory counter introduced by the perf optimisation. /// @@ -713,8 +836,8 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Enqueue_and_drain_keep_queue_depth_consistent_with_storage() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink( - _dbPath, writer, _log, batchSize: 5, capacity: 8); + using var sink = new LocalDbStoreAndForwardSink( + Db, writer, _log, batchSize: 5, capacity: 8); // Burst-enqueue below capacity — the in-memory counter must stay aligned with the // SELECT COUNT(*) that GetStatus runs. @@ -760,7 +883,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Counter_remains_consistent_under_concurrent_enqueue_and_drain() { var writer = new FakeWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); var enqueuers = Enumerable.Range(0, 3).Select(t => Task.Run(async () => { @@ -790,7 +913,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable /// Regression for Core.AlarmHistorian-012: when WriteBatchAsync throws /// the drain exits via re-throw /// without resetting _drainState, leaving the status surface permanently - /// showing Draining. The next call to + /// showing Draining. The next call to /// must reflect Idle or BackingOff, not the stale Draining /// state that was set at the top of the drain tick. /// @@ -798,7 +921,7 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable public async Task Drain_cancelled_mid_write_resets_drain_state_to_not_draining() { var writer = new CancellableWriter(); - using var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); await sink.EnqueueAsync(Event("A1"), CancellationToken.None); using var cts = new CancellationTokenSource(); @@ -840,49 +963,68 @@ public sealed class SqliteStoreAndForwardSinkTests : IDisposable /// COUNT(*) read directly from storage — proves the in-memory counter has not /// drifted from the persisted truth. /// - private void AssertQueueDepthMatchesStorage(SqliteStoreAndForwardSink sink) + private void AssertQueueDepthMatchesStorage(LocalDbStoreAndForwardSink sink) { - using var conn = new SqliteConnection($"Data Source={_dbPath}"); - conn.Open(); + using var conn = Db.CreateConnection(); using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM Queue WHERE DeadLettered = 0"; + cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0"; var live = (long)(cmd.ExecuteScalar() ?? 0L); sink.GetStatus().QueueDepth.ShouldBe(live, "GetStatus must agree with a fresh COUNT(*)"); } /// - /// Regression for Core.AlarmHistorian-014: - /// and must throw - /// after + /// Regression for Core.AlarmHistorian-014: + /// and must throw + /// after /// is called, consistent with the guards already present on - /// and - /// . + /// and + /// . /// [Fact] public void Disposed_sink_throws_ObjectDisposedException_on_GetStatus_and_RetryDeadLettered() { var writer = new FakeWriter(); - var sink = new SqliteStoreAndForwardSink(_dbPath, writer, _log); + var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); sink.Dispose(); Should.Throw(() => sink.GetStatus()); Should.Throw(() => sink.RetryDeadLettered()); } - /// Insert a queue row whose PayloadJson cannot deserialize into an AlarmHistorianEvent. - private void InsertCorruptRow(string alarmId) + /// + /// Insert a queue row whose payload cannot deserialize into an AlarmHistorianEvent, stamped + /// at an explicit enqueue time. + /// + /// + /// The timestamp is a parameter rather than DateTime.UtcNow because the drain reads + /// in enqueued_at_utc order, and the tests that use this helper care precisely about + /// where the corrupt row sits in that order. Three unsynchronised clock reads can land on + /// one tick, which would make the interleaving — and therefore the test — a coin flip. + /// + private void InsertCorruptRow(string alarmId, DateTime enqueuedAt) { - using var conn = new SqliteConnection($"Data Source={_dbPath}"); - conn.Open(); + using var conn = Db.CreateConnection(); using var cmd = conn.CreateCommand(); cmd.CommandText = """ - INSERT INTO Queue (AlarmId, EnqueuedUtc, PayloadJson, AttemptCount) - VALUES ($alarmId, $enqueued, $payload, 0); + INSERT INTO alarm_sf_events (id, alarm_id, enqueued_at_utc, payload_json, attempt_count) + VALUES ($id, $alarmId, $enqueued, $payload, 0); """; + cmd.Parameters.AddWithValue("$id", Guid.NewGuid().ToString("N")); cmd.Parameters.AddWithValue("$alarmId", alarmId); - cmd.Parameters.AddWithValue("$enqueued", DateTime.UtcNow.ToString("O")); + cmd.Parameters.AddWithValue("$enqueued", enqueuedAt.ToString("O")); // JSON literal "null" round-trips through Deserialize as a null reference. cmd.Parameters.AddWithValue("$payload", "null"); cmd.ExecuteNonQuery(); } + + /// + /// Monotonic test clock — every read advances one second, so enqueue order is total and + /// the drain's ORDER BY enqueued_at_utc is deterministic. + /// + private sealed class TickingClock + { + private DateTime _now = new(2026, 7, 21, 0, 0, 0, DateTimeKind.Utc); + + public DateTime Next() => _now = _now.AddSeconds(1); + } } diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests.csproj b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests.csproj index 0cb9db4c..3f567086 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests.csproj +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests.csproj @@ -17,6 +17,14 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs new file mode 100644 index 00000000..b5c3ed6b --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs @@ -0,0 +1,132 @@ +using Akka.Actor; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; +using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; + +/// +/// The bridge that carries the Primary-gate decision out of the actor and into +/// , where the alarm store-and-forward drain can read it. +/// +/// +/// +/// The drain runs on a timer owned by the sink, not on a mailbox, so it cannot receive +/// itself — but it must be Primary-scoped, because +/// the queue it drains replicates to the pair peer and two draining nodes would deliver +/// every alarm event twice. +/// +/// +/// These cases mirror 's matrix on purpose: +/// the view must reach exactly the same verdict as the inbound-write and native-ack gates, +/// because a second, subtly different notion of "am I the Primary" is how a pair ends up +/// with one node writing and neither draining. +/// +/// +public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase +{ + private static readonly NodeId TestNode = NodeId.Parse("driver-roleview-test"); + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5); + + /// + /// Before any snapshot arrives the view must read open, not closed. + /// + /// + /// A deployment that runs no redundancy never publishes here at all. Defaulting closed + /// would silently stop its alarm history forever — the exact failure the drain gate exists + /// to avoid causing. + /// + [Fact] + public void An_unpublished_view_reads_as_primary() + { + new RedundancyRoleView().ShouldServiceAsPrimary.ShouldBeTrue(); + } + + [Fact] + public void Secondary_role_closes_the_view() + { + var view = SpawnAndTellRole(RedundancyRole.Secondary, driverMemberCount: 2); + + AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeFalse(), duration: Timeout); + } + + [Fact] + public void Primary_role_opens_the_view() + { + var view = SpawnAndTellRole(RedundancyRole.Primary, driverMemberCount: 2); + + AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeTrue(), duration: Timeout); + } + + /// Unknown role + a real driver peer ⇒ closed, matching the write gate's default-DENY. + [Fact] + public void Unknown_role_with_a_driver_peer_closes_the_view() + { + // A snapshot that never names this node is how the role stays unknown in practice — the + // documented identity-mismatch shape, not a contrived case. + var view = SpawnAndTellForeignSnapshot(driverMemberCount: 2); + + AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeFalse(), duration: Timeout); + } + + /// Unknown role + no driver peer ⇒ open, preserving the single-node posture. + [Fact] + public void Unknown_role_without_a_driver_peer_leaves_the_view_open() + { + var view = SpawnAndTellForeignSnapshot(driverMemberCount: 1); + + AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeTrue(), duration: Timeout); + } + + /// A demotion must move the view, not just an initial promotion. + [Fact] + public void A_demotion_closes_a_previously_open_view() + { + var view = new RedundancyRoleView(); + var host = SpawnHost(view, driverMemberCount: 2); + + host.Tell(Snapshot(TestNode, RedundancyRole.Primary)); + AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeTrue(), duration: Timeout); + + host.Tell(Snapshot(TestNode, RedundancyRole.Secondary)); + AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeFalse(), duration: Timeout); + } + + // ---------------- helpers ---------------- + + private RedundancyRoleView SpawnAndTellRole(RedundancyRole role, int driverMemberCount) + { + var view = new RedundancyRoleView(); + SpawnHost(view, driverMemberCount).Tell(Snapshot(TestNode, role)); + return view; + } + + private RedundancyRoleView SpawnAndTellForeignSnapshot(int driverMemberCount) + { + var view = new RedundancyRoleView(); + SpawnHost(view, driverMemberCount).Tell(Snapshot(NodeId.Parse("some-other-node"), RedundancyRole.Primary)); + return view; + } + + private IActorRef SpawnHost(IRedundancyRoleView view, int driverMemberCount) => + Sys.ActorOf(DriverHostActor.Props( + NewInMemoryDbFactory(), + TestNode, + coordinator: null, + driverMemberCountProvider: () => driverMemberCount, + redundancyRoleView: view)); + + private static RedundancyStateChanged Snapshot(NodeId node, RedundancyRole role) => + new( + [ + new NodeRedundancyState(node, role, + IsClusterLeader: role == RedundancyRole.Primary, + IsRoleLeaderForDriver: role == RedundancyRole.Primary, + AsOfUtc: DateTime.UtcNow), + ], + CorrelationId.NewId()); +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs index 8100f19f..59d375e7 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs @@ -1,8 +1,10 @@ +using Microsoft.Data.Sqlite; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Shouldly; using Xunit; +using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; using ZB.MOM.WW.OtOpcUa.Runtime.Historian; @@ -11,12 +13,12 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Historian; /// /// Verifies the config-gated AddAlarmHistorian registration: when the /// AlarmHistorian section is absent or disabled the -/// default survives; when it is enabled a real wins +/// default survives; when it is enabled a real wins /// (last-registration-wins over the TryAddSingleton Null default). /// public sealed class AlarmHistorianRegistrationTests { - /// A no-op writer the factory hands the Sqlite sink; never actually invoked in these tests. + /// A no-op writer the factory hands the sink; never actually invoked in these tests. private sealed class FakeWriter : IAlarmHistorianWriter { public Task> WriteBatchAsync( @@ -65,34 +67,61 @@ public sealed class AlarmHistorianRegistrationTests } [Fact] - public void Section_enabled_registers_sqlite_sink() + public void Section_enabled_registers_the_localdb_sink() { var tempDir = Path.Combine(Path.GetTempPath(), "otopcua-alarmhist-test-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(tempDir); - var dbPath = Path.Combine(tempDir, "alarm-historian-test.db"); + var dbPath = Path.Combine(tempDir, "node-local.db"); try { - var services = BaseServices(); var config = ConfigFrom(new Dictionary { ["AlarmHistorian:Enabled"] = "true", - ["AlarmHistorian:DatabasePath"] = dbPath, + ["LocalDb:Path"] = dbPath, }); + var services = BaseServices(); + services.AddZbLocalDb(config, db => + { + using var connection = db.CreateConnection(); + AlarmSfSchema.Apply(connection); + db.RegisterReplicated(AlarmSfSchema.EventsTable); + }); services.AddAlarmHistorian(config, (_, _) => new FakeWriter()); using (var provider = services.BuildServiceProvider()) { - provider.GetRequiredService().ShouldBeOfType(); + provider.GetRequiredService().ShouldBeOfType(); } // dispose stops the drain loop + releases the SQLite file handle } finally { + SqliteConnection.ClearAllPools(); try { Directory.Delete(tempDir, recursive: true); } catch (IOException) { /* best effort */ } } } + /// + /// An enabled historian with no LocalDb registered must fail at resolution, loudly. + /// + /// + /// The queue has nowhere to live without it. Falling back to the Null sink here would look + /// like a working historian while silently discarding every alarm event — the failure mode + /// this whole registration is meant to make impossible. + /// + [Fact] + public void Section_enabled_without_localdb_throws_rather_than_silently_discarding() + { + var services = BaseServices(); + var config = ConfigFrom(new Dictionary { ["AlarmHistorian:Enabled"] = "true" }); + + services.AddAlarmHistorian(config, (_, _) => new FakeWriter()); + + using var provider = services.BuildServiceProvider(); + Should.Throw(() => provider.GetRequiredService()); + } + [Fact] public void Section_binds_drain_capacity_and_retention_knobs() { @@ -112,17 +141,10 @@ public sealed class AlarmHistorianRegistrationTests opts.DeadLetterRetentionDays.ShouldBe(7); } - [Fact] - public void Validate_warns_on_relative_database_path_when_enabled() - { - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "alarm-historian.db" }; - opts.Validate().ShouldContain(w => w.Contains("DatabasePath")); - } - [Fact] public void Validate_is_silent_when_correctly_configured() { - new AlarmHistorianOptions { Enabled = true, DatabasePath = "/abs/h.db" }.Validate().ShouldBeEmpty(); + new AlarmHistorianOptions { Enabled = true }.Validate().ShouldBeEmpty(); } [Fact] @@ -134,39 +156,39 @@ public sealed class AlarmHistorianRegistrationTests [Fact] public void Validate_warns_on_non_positive_drain_interval() { - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "/abs/h.db", DrainIntervalSeconds = 0 }; + var opts = new AlarmHistorianOptions { Enabled = true, DrainIntervalSeconds = 0 }; opts.Validate().ShouldContain(w => w.Contains("DrainIntervalSeconds")); } [Fact] public void Validate_warns_on_non_positive_capacity() { - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "/abs/h.db", Capacity = 0 }; + var opts = new AlarmHistorianOptions { Enabled = true, Capacity = 0 }; opts.Validate().ShouldContain(w => w.Contains("Capacity")); } [Fact] public void Validate_warns_on_non_positive_retention() { - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "/abs/h.db", DeadLetterRetentionDays = 0 }; + var opts = new AlarmHistorianOptions { Enabled = true, DeadLetterRetentionDays = 0 }; opts.Validate().ShouldContain(w => w.Contains("DeadLetterRetentionDays")); } [Fact] public void Validate_warns_on_non_positive_max_attempts() { - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "/abs/h.db", MaxAttempts = 0 }; + var opts = new AlarmHistorianOptions { Enabled = true, MaxAttempts = 0 }; opts.Validate().ShouldContain(w => w.Contains("MaxAttempts")); } [Fact] public void Validate_accumulates_multiple_warnings() { - // relative path + non-positive drain interval ⇒ both warnings, not short-circuited on the first. - var opts = new AlarmHistorianOptions { Enabled = true, DatabasePath = "alarm-historian.db", DrainIntervalSeconds = 0 }; + // Two bad knobs ⇒ both warnings, not short-circuited on the first. + var opts = new AlarmHistorianOptions { Enabled = true, DrainIntervalSeconds = 0, Capacity = 0 }; var warnings = opts.Validate(); - warnings.ShouldContain(w => w.Contains("DatabasePath")); warnings.ShouldContain(w => w.Contains("DrainIntervalSeconds")); + warnings.ShouldContain(w => w.Contains("Capacity")); warnings.Count.ShouldBeGreaterThanOrEqualTo(2); } } From af545efdf547b565960164940a45604118f44ef7 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 04:31:10 -0400 Subject: [PATCH 04/13] feat(localdb): one-time alarm-historian.db migrator Copies the pre-consolidation store-and-forward queue into the consolidated database on first boot, then renames the legacy file aside. Rows are in that file precisely because the historian could not be reached, so dropping them on upgrade would discard exactly the alarm audit trail the queue exists to protect. Runs last in OnReady, after every RegisterReplicated call. It is the only thing in OnReady that writes rows, and capture is trigger-based: a migration that ran before registration would recover the backlog locally and never replicate a line of it, silently and permanently. Ids are derived from the payload rather than the plan's mig-{node}-{legacyId} scheme. Node-prefixing solves the collision the legacy AUTOINCREMENT key would cause -- node A's row 7 and node B's row 7 are different alarms -- but it preserves a duplication that should be collapsed instead. A warm pair's two legacy files OVERLAP: HistorianAdapterActor default-writes while its redundancy role is unknown, so both nodes accepted the same transitions during every boot window. Prefixed ids would carry those duplicates into the merged buffer forever; equal-payload ids converge them. The same property makes a crash between commit and rename harmless under INSERT OR IGNORE. OnReady now takes IConfiguration rather than offering an overload that skips the migration. A wiring mistake that silently discarded a node's undelivered alarm history is not a mistake worth making possible. The copy is restricted to the columns the legacy table actually has. Naming a column an older build never wrote throws "no such column", which would discard every row in the table rather than the one field. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- ...7-20-localdb-adoption-phase2.md.tasks.json | 5 +- .../AlarmSfSchema.cs | 28 ++ .../LocalDbStoreAndForwardSink.cs | 19 +- .../Configuration/AlarmSfLegacyMigrator.cs | 217 ++++++++++++ .../Configuration/LocalDbRegistration.cs | 2 +- .../Configuration/LocalDbSetup.cs | 16 +- .../LocalDb/AlarmSfLegacyMigratorTests.cs | 333 ++++++++++++++++++ .../LocalDb/LocalDbPairHarness.cs | 2 +- .../LocalDbSetupTests.cs | 2 +- 9 files changed, 598 insertions(+), 26 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/AlarmSfLegacyMigrator.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfLegacyMigratorTests.cs diff --git a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json index 0272fcc7..512b83cc 100644 --- a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json +++ b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json @@ -37,10 +37,11 @@ { "id": 4, "subject": "Task 4: One-time alarm-historian.db legacy migrator", - "status": "pending", + "status": "completed", "blockedBy": [ 3 - ] + ], + "note": "AlarmSfLegacyMigrator runs LAST in OnReady (which now takes IConfiguration - no skip-migration overload exists). DEVIATION D-6: ids are the payload hash (AlarmSfSchema.DeriveId, lifted out of the sink) rather than mig-{node}-{legacyId} - a warm pair's two legacy files OVERLAP, and node-prefixing would carry that duplication forward forever. DEVIATION: tests live in Host.IntegrationTests; the plan's Host.Tests project does not exist." }, { "id": 5, diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs index eaf4553c..7705dc02 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmSfSchema.cs @@ -1,3 +1,5 @@ +using System.Security.Cryptography; +using System.Text; using Microsoft.Data.Sqlite; namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; @@ -35,6 +37,32 @@ public static class AlarmSfSchema /// The single primary-key column. public const string IdColumn = "id"; + /// + /// Derives a row's primary key from its serialized payload. + /// + /// + /// + /// Deterministic rather than a fresh GUID, so that the same event arriving twice + /// converges to one row under last-writer-wins instead of duplicating. That happens for + /// real in two places: HistorianAdapterActor default-writes while its redundancy + /// role is unknown, so both nodes of a pair accept the same fanned transition in the + /// window before the first snapshot; and both nodes independently migrate their own + /// legacy queue file, which for a warm pair holds overlapping history. + /// + /// + /// Two genuinely distinct events cannot collide. AlarmHistorianEvent carries a + /// full-precision timestamp alongside the alarm id, transition kind, message and user, + /// so an equal hash means an equal event. + /// + /// + /// The serialized AlarmHistorianEvent. + /// The row's primary key. + public static string DeriveId(string payloadJson) + { + ArgumentNullException.ThrowIfNull(payloadJson); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(payloadJson))); + } + /// /// Creates the buffer table if it does not already exist. Idempotent. /// diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs index 17285524..8c696694 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs @@ -1,5 +1,3 @@ -using System.Security.Cryptography; -using System.Text; using System.Text.Json; using Serilog; using ZB.MOM.WW.LocalDb; @@ -213,21 +211,6 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl catch (ObjectDisposedException) { /* raced with Dispose — nothing to re-arm */ } } - /// - /// Derives the row's primary key from its payload. - /// - /// - /// Deterministic rather than a fresh GUID so that the same event enqueued independently on - /// both nodes of a pair converges to one row under last-writer-wins instead of duplicating. - /// That happens for real: HistorianAdapterActor default-writes while its redundancy - /// role is unknown, so in the window before the first snapshot arrives both adapters accept - /// the same fanned transition. Two distinct events cannot collide — - /// carries a full-precision timestamp alongside the alarm - /// id, kind, message and user, so an equal hash means an equal event. - /// - private static string DeriveId(string payloadJson) => - Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(payloadJson))); - /// public async Task EnqueueAsync(AlarmHistorianEvent evt, CancellationToken cancellationToken) { @@ -249,7 +232,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl """, new { - Id = DeriveId(payload), + Id = AlarmSfSchema.DeriveId(payload), AlarmId = evt.AlarmId, EnqueuedAtUtc = _clock().ToString("O"), PayloadJson = payload, diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/AlarmSfLegacyMigrator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/AlarmSfLegacyMigrator.cs new file mode 100644 index 00000000..8e6cc31b --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/AlarmSfLegacyMigrator.cs @@ -0,0 +1,217 @@ +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Configuration; +using ZB.MOM.WW.LocalDb; +using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; + +namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; + +/// +/// One-time copy of the pre-consolidation alarm-historian.db store-and-forward queue +/// into the consolidated ZB.MOM.WW.LocalDb database. +/// +/// +/// +/// What is at stake. Rows sit in that file precisely because the historian could not +/// be reached. Discarding them on upgrade would throw away exactly the alarm audit trail +/// the store-and-forward queue exists to protect — so the copy tolerates an older column +/// set rather than bailing out, and refuses to rename a file it did not actually read. +/// +/// +/// Runs after RegisterReplicated, deliberately. Capture is trigger-based, so +/// rows inserted before registration never enter the oplog and never reach the peer — +/// silently, and permanently, because nothing revisits history. Migrating after +/// registration means the recovered backlog replicates like any other write. +/// +/// +/// Ids come from the payload, not from the legacy row id. The legacy primary key was +/// RowId INTEGER PRIMARY KEY AUTOINCREMENT, which cannot survive into a replicated +/// table: each node allocated its own sequence, so node A's row 7 and node B's row 7 are +/// different alarms that would silently overwrite one another under last-writer-wins. +/// Hashing the payload () solves that and one more +/// problem besides — a warm pair's two legacy files overlap, because +/// HistorianAdapterActor default-writes while its redundancy role is unknown and +/// both nodes therefore accepted the same transitions during every boot window. A +/// node-prefixed id would carry that duplication into the merged buffer forever; an +/// equal-payload id collapses it. It is also what makes a re-run harmless under +/// INSERT OR IGNORE. +/// +/// +/// All-or-nothing. The copy runs in one transaction and the legacy file is renamed +/// to <name>.migrated only after the commit. A failure throws out of +/// OnReady, failing host startup with the legacy file untouched — no half-migrated +/// state to reason about, and the operator still holds the original. A later boot sees the +/// renamed file and no-ops. +/// +/// +public static class AlarmSfLegacyMigrator +{ + private const string MigratedSuffix = ".migrated"; + + /// The legacy queue table. + private const string LegacyTable = "Queue"; + + /// + /// The configuration key that used to carry the standalone queue file's path. Read as a raw + /// key because the corresponding AlarmHistorianOptions property was removed with the + /// bespoke file management it configured. + /// + public const string LegacyPathKey = "AlarmHistorian:DatabasePath"; + + /// The pre-removal default for . + private const string DefaultLegacyPath = "alarm-historian.db"; + + /// + /// The legacy column whose absence means this is not a queue file we can read. Without the + /// payload there is no event and no id to derive from one. + /// + private const string RequiredColumn = "PayloadJson"; + + /// + /// Every legacy column, mapped to its consolidated counterpart. Columns absent from an + /// older file are dropped from the copy rather than failing it — see + /// . + /// + private static readonly (string Legacy, string Current)[] ColumnMap = + [ + ("AlarmId", "alarm_id"), + ("EnqueuedUtc", "enqueued_at_utc"), + ("PayloadJson", "payload_json"), + ("AttemptCount", "attempt_count"), + ("LastAttemptUtc", "last_attempt_utc"), + ("LastError", "last_error"), + ("DeadLettered", "dead_lettered"), + ]; + + /// + /// Copies any legacy alarm queue into , then renames the legacy file. + /// + /// The consolidated database, with alarm_sf_events already registered. + /// Configuration supplying the legacy queue's path. + public static void Migrate(ILocalDb db, IConfiguration config) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(config); + + var legacyPath = ResolveLegacyPath(config); + if (!ShouldMigrate(legacyPath)) return; + + using (var legacy = OpenLegacyReadOnly(legacyPath)) + { + var present = PresentColumns(legacy); + + // An absent table probes as zero columns, so this one guard covers both "no queue + // table" and "a shape we do not recognise". Returning without renaming leaves the file + // for an operator to inspect. + if (!present.Contains(RequiredColumn)) return; + + using var connection = db.CreateConnection(); + using var transaction = connection.BeginTransaction(); + CopyRows(legacy, connection, transaction, present); + transaction.Commit(); + } + + // Only after the commit. A crash between the two leaves the file in place and the next + // boot copies it again, which the payload-derived ids make harmless. + File.Move(legacyPath, legacyPath + MigratedSuffix, overwrite: true); + } + + /// + /// Resolves the legacy queue path from the removed configuration key, falling back to the + /// code default it used to carry. + /// + /// + /// Relative paths resolve against the process working directory — not the LocalDb + /// directory — because that is where the old code actually put them. + /// + /// The application configuration. + /// An absolute path, or empty when there is nothing durable to migrate from. + internal static string ResolveLegacyPath(IConfiguration config) + { + var path = config[LegacyPathKey] ?? DefaultLegacyPath; + + // In-memory and URI-form sources (test / dev configurations) have nothing durable behind + // them, and Path.GetFullPath on them would produce nonsense. + if (string.IsNullOrWhiteSpace(path) || + path.Equals(":memory:", StringComparison.OrdinalIgnoreCase) || + path.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + { + return string.Empty; + } + + return Path.GetFullPath(path); + } + + /// Whether there is a legacy file worth opening. + private static bool ShouldMigrate(string legacyPath) => + !string.IsNullOrEmpty(legacyPath) + && File.Exists(legacyPath) + && !File.Exists(legacyPath + MigratedSuffix); + + /// Opens the legacy file read-only, so a failed migration cannot damage it. + private static SqliteConnection OpenLegacyReadOnly(string path) + { + var connection = new SqliteConnection(new SqliteConnectionStringBuilder + { + DataSource = path, + Mode = SqliteOpenMode.ReadOnly, + }.ToString()); + connection.Open(); + return connection; + } + + /// + /// Which of the columns we know about the legacy table actually has. + /// + /// + /// A file written by an older build predates some columns, and naming a missing one in the + /// SELECT throws "no such column" — which discards every row in the table rather than the + /// one field. Intersecting first means an old file migrates its data and simply leaves the + /// newer columns at their schema defaults. + /// + private static HashSet PresentColumns(SqliteConnection legacy) + { + var present = new HashSet(StringComparer.OrdinalIgnoreCase); + + using var cmd = legacy.CreateCommand(); + cmd.CommandText = $"SELECT name FROM pragma_table_info('{LegacyTable}')"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + present.Add(reader.GetString(0)); + + return present; + } + + /// Copies every legacy row into the consolidated table, keyed by payload hash. + private static void CopyRows( + SqliteConnection legacy, + SqliteConnection target, + SqliteTransaction transaction, + HashSet present) + { + var columns = ColumnMap.Where(c => present.Contains(c.Legacy)).ToArray(); + var payloadOrdinal = Array.FindIndex(columns, c => c.Legacy == RequiredColumn); + + using var read = legacy.CreateCommand(); + read.CommandText = + $"SELECT {string.Join(", ", columns.Select(c => c.Legacy))} FROM {LegacyTable}"; + + using var reader = read.ExecuteReader(); + while (reader.Read()) + { + using var insert = target.CreateCommand(); + insert.Transaction = transaction; + insert.CommandText = $""" + INSERT OR IGNORE INTO {AlarmSfSchema.EventsTable} + ({AlarmSfSchema.IdColumn}, {string.Join(", ", columns.Select(c => c.Current))}) + VALUES + ($id, {string.Join(", ", columns.Select((_, i) => $"$c{i}"))}) + """; + + insert.Parameters.AddWithValue("$id", AlarmSfSchema.DeriveId(reader.GetString(payloadOrdinal))); + for (var i = 0; i < columns.Length; i++) + insert.Parameters.AddWithValue($"$c{i}", reader.GetValue(i) ?? DBNull.Value); + + insert.ExecuteNonQuery(); + } + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbRegistration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbRegistration.cs index c23fa6bc..aa7522ec 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbRegistration.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbRegistration.cs @@ -88,7 +88,7 @@ public static class LocalDbRegistration ArgumentNullException.ThrowIfNull(services); ArgumentNullException.ThrowIfNull(configuration); - services.AddZbLocalDb(configuration, LocalDbSetup.OnReady); + services.AddZbLocalDb(configuration, db => LocalDbSetup.OnReady(db, configuration)); services.AddZbLocalDbReplication(configuration); // The consumer-facing seam. WithOtOpcUaRuntimeActors resolves this optionally and threads it diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs index 2819ea17..ecbd6c8a 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Configuration; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache; @@ -27,8 +28,8 @@ public static class LocalDbSetup /// RegisterReplicated is what installs the three AFTER triggers that capture /// changes into the oplog. Any row written before that call is never captured, so it /// never reaches the peer — silently, and permanently, because nothing ever revisits - /// history. The Phase-2 legacy alarm migrator therefore runs after every - /// registration, not alongside the DDL. + /// history. therefore runs last, after every + /// registration — it writes rows, and they must be captured like any other write. /// /// /// The alarm buffer's tables are created unconditionally, regardless of whether this @@ -39,9 +40,15 @@ public static class LocalDbSetup /// /// /// The freshly constructed local database. - public static void OnReady(ILocalDb db) + /// + /// Application configuration, read by the legacy migrator for the pre-consolidation queue's + /// path. Required rather than optional: an overload that silently skipped the migration + /// would be one wiring mistake away from discarding a node's undelivered alarm history. + /// + public static void OnReady(ILocalDb db, IConfiguration configuration) { ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(configuration); // CreateConnection() hands back an already-open, pragma-configured connection carrying the // zb_hlc_next() UDF the capture triggers need. Calling Open() on it would throw. @@ -54,5 +61,8 @@ public static class LocalDbSetup db.RegisterReplicated(DeploymentCacheSchema.ArtifactsTable); db.RegisterReplicated(DeploymentCacheSchema.PointerTable); db.RegisterReplicated(AlarmSfSchema.EventsTable); + + // LAST, and only here. This is the one call in OnReady that writes rows. + AlarmSfLegacyMigrator.Migrate(db, configuration); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfLegacyMigratorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfLegacyMigratorTests.cs new file mode 100644 index 00000000..0a581633 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfLegacyMigratorTests.cs @@ -0,0 +1,333 @@ +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; +using Xunit; +using ZB.MOM.WW.LocalDb; +using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.LocalDb; + +/// +/// The one-time copy of the pre-consolidation alarm-historian.db queue into the +/// consolidated LocalDb. +/// +/// +/// +/// What is being protected is undelivered alarm history: rows that are in the legacy file +/// precisely because the historian could not be reached. Dropping them on upgrade would +/// discard exactly the audit trail the store-and-forward queue exists to preserve. +/// +/// +/// Run against a real temp-file built through the production +/// , so the registration order the migrator depends on is +/// the one the host actually runs — a hand-written schema here would prove only that the +/// test agrees with itself. +/// +/// +public sealed class AlarmSfLegacyMigratorTests : IDisposable +{ + private readonly string _dbPath = + Path.Combine(Path.GetTempPath(), $"otopcua-alarmsf-mig-{Guid.NewGuid():N}.db"); + + private readonly string _legacyPath = + Path.Combine(Path.GetTempPath(), $"otopcua-alarmsf-legacy-{Guid.NewGuid():N}.db"); + + private ServiceProvider? _provider; + + // ---- the cases --------------------------------------------------------------------------- + + [Fact] + public async Task Every_legacy_row_lands_in_the_consolidated_table() + { + SeedLegacy( + (1, "eq/a", Payload("eq/a"), 0, dead: 0), + (2, "eq/b", Payload("eq/b"), 3, dead: 0), + (3, "eq/c", Payload("eq/c"), 9, dead: 1)); + + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + (await CountAsync(db)).ShouldBe(3); + // Attempt counts and the dead-letter flag must survive: a migrated row that lost its + // attempt count would restart its retry budget, and one that lost its dead-letter flag + // would be re-sent to a historian that already refused it permanently. + var rows = await db.QueryAsync( + "SELECT alarm_id, attempt_count, dead_lettered FROM alarm_sf_events ORDER BY alarm_id", + r => (AlarmId: r.GetString(0), Attempts: r.GetInt64(1), Dead: r.GetInt64(2)), + parameters: null, + TestContext.Current.CancellationToken); + + rows.ShouldBe([("eq/a", 0L, 0L), ("eq/b", 3L, 0L), ("eq/c", 9L, 1L)]); + } + + [Fact] + public async Task Migrated_rows_enter_the_oplog() + { + // THE ordering pin. Capture is trigger-based, so rows copied in before + // RegisterReplicated would never enter the oplog and would never reach the peer — + // silently, and permanently. A migrator invoked too early passes every other case here. + SeedLegacy((1, "eq/a", Payload("eq/a"), 0, dead: 0)); + + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + var oplog = await db.QueryAsync( + "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'alarm_sf_events'", + r => r.GetInt64(0), + parameters: null, + TestContext.Current.CancellationToken); + + oplog[0].ShouldBe(1); + } + + [Fact] + public async Task A_second_run_is_a_no_op() + { + SeedLegacy((1, "eq/a", Payload("eq/a"), 0, dead: 0)); + var db = BuildDb(); + var config = ConfigWithLegacyPath(); + + AlarmSfLegacyMigrator.Migrate(db, config); + AlarmSfLegacyMigrator.Migrate(db, config); + + (await CountAsync(db)).ShouldBe(1); + File.Exists(_legacyPath).ShouldBeFalse("the legacy file is renamed once the copy commits"); + File.Exists(_legacyPath + ".migrated").ShouldBeTrue(); + } + + [Fact] + public async Task Re_migrating_the_same_rows_cannot_duplicate_them() + { + // The sidecar rename is the normal guard, but it is not the only one that has to hold: + // a crash between commit and rename leaves the file in place, and the next boot copies it + // again. Ids are derived from the payload, so the re-copy collides with itself and is + // ignored rather than doubling the queue. + SeedLegacy((1, "eq/a", Payload("eq/a"), 0, dead: 0)); + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + File.Move(_legacyPath + ".migrated", _legacyPath); // simulate the crash-before-rename + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + (await CountAsync(db)).ShouldBe(1); + } + + [Fact] + public async Task An_older_legacy_file_missing_a_column_still_migrates() + { + // A file written by an older build predates LastError. Naming a missing column in the + // SELECT throws "no such column", which would discard every row in the table rather than + // the one field. Intersecting the column list first means the data migrates and the + // absent column keeps its schema default. + SeedLegacyWithoutLastError((1, "eq/a", Payload("eq/a"))); + + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + (await CountAsync(db)).ShouldBe(1); + } + + [Fact] + public async Task A_failed_copy_leaves_the_legacy_file_untouched() + { + // The legacy table exists but its payload column is missing, so the required-column guard + // rejects the file. Nothing is copied and nothing is renamed: the operator still has the + // original to recover from, which is the whole point of renaming only after commit. + SeedUnrecognisedLegacy(); + + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + (await CountAsync(db)).ShouldBe(0); + File.Exists(_legacyPath).ShouldBeTrue("an un-copied file must not be renamed away"); + File.Exists(_legacyPath + ".migrated").ShouldBeFalse(); + } + + [Fact] + public async Task A_missing_legacy_file_is_a_clean_no_op() + { + // The expected case on a fresh install and on any node that never enabled the historian. + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + (await CountAsync(db)).ShouldBe(0); + } + + [Theory] + [InlineData(":memory:")] + [InlineData("file:thing?mode=memory")] + [InlineData("")] + public async Task Non_file_legacy_sources_are_no_ops(string configured) + { + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, Config(configured)); + + (await CountAsync(db)).ShouldBe(0); + } + + [Fact] + public async Task Two_nodes_migrating_the_same_event_converge_on_one_row() + { + // A warm pair's two legacy files overlap: HistorianAdapterActor default-writes while the + // redundancy role is unknown, so both nodes accepted the same transitions during every + // boot window. Node-prefixed ids would carry that duplication forward into the merged + // buffer permanently; payload-derived ids collapse it. + var shared = Payload("eq/shared"); + SeedLegacy((1, "eq/shared", shared, 0, dead: 0)); + + var db = BuildDb(); + AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath()); + + // The peer's file: a different legacy RowId for the very same event. + var peerLegacy = _legacyPath + ".peer"; + SeedLegacyAt(peerLegacy, (77, "eq/shared", shared, 0, dead: 0)); + AlarmSfLegacyMigrator.Migrate(db, Config(peerLegacy)); + + (await CountAsync(db)).ShouldBe(1); + + try { File.Delete(peerLegacy + ".migrated"); } catch { /* best effort */ } + } + + // ---- fixture ----------------------------------------------------------------------------- + + private ILocalDb BuildDb() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["LocalDb:Path"] = _dbPath }) + .Build(); + + _provider = new ServiceCollection() + .AddZbLocalDb(configuration, db => LocalDbSetup.OnReady(db, configuration)) + .BuildServiceProvider(); + + return _provider.GetRequiredService(); + } + + private IConfiguration ConfigWithLegacyPath() => Config(_legacyPath); + + private static IConfiguration Config(string legacyPath) => + new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["AlarmHistorian:DatabasePath"] = legacyPath, + }) + .Build(); + + private static async Task CountAsync(ILocalDb db) + { + var rows = await db.QueryAsync( + "SELECT COUNT(*) FROM alarm_sf_events", + r => r.GetInt64(0), + parameters: null, + TestContext.Current.CancellationToken); + return rows[0]; + } + + /// A payload shaped like a serialized AlarmHistorianEvent. + private static string Payload(string alarmId) => + $$"""{"AlarmId":"{{alarmId}}","EquipmentPath":"line/eq","AlarmName":"hi","TimestampUtc":"2026-07-20T00:00:00Z"}"""; + + private void SeedLegacy(params (long RowId, string AlarmId, string Payload, long Attempts, long dead)[] rows) => + SeedLegacyAt(_legacyPath, rows); + + /// Creates a legacy queue file with the full pre-cutover schema and the given rows. + private static void SeedLegacyAt( + string path, params (long RowId, string AlarmId, string Payload, long Attempts, long dead)[] rows) + { + using var conn = Open(path); + Exec(conn, """ + CREATE TABLE Queue ( + RowId INTEGER PRIMARY KEY AUTOINCREMENT, + AlarmId TEXT NOT NULL, + EnqueuedUtc TEXT NOT NULL, + PayloadJson TEXT NOT NULL, + AttemptCount INTEGER NOT NULL DEFAULT 0, + LastAttemptUtc TEXT NULL, + LastError TEXT NULL, + DeadLettered INTEGER NOT NULL DEFAULT 0 + ); + """); + + foreach (var row in rows) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + INSERT INTO Queue (RowId, AlarmId, EnqueuedUtc, PayloadJson, AttemptCount, DeadLettered) + VALUES ($id, $alarm, '2026-07-20T00:00:00.0000000Z', $payload, $attempts, $dead) + """; + cmd.Parameters.AddWithValue("$id", row.RowId); + cmd.Parameters.AddWithValue("$alarm", row.AlarmId); + cmd.Parameters.AddWithValue("$payload", row.Payload); + cmd.Parameters.AddWithValue("$attempts", row.Attempts); + cmd.Parameters.AddWithValue("$dead", row.dead); + cmd.ExecuteNonQuery(); + } + } + + /// A legacy file from an older build, predating the LastError column. + private void SeedLegacyWithoutLastError(params (long RowId, string AlarmId, string Payload)[] rows) + { + using var conn = Open(_legacyPath); + Exec(conn, """ + CREATE TABLE Queue ( + RowId INTEGER PRIMARY KEY AUTOINCREMENT, + AlarmId TEXT NOT NULL, + EnqueuedUtc TEXT NOT NULL, + PayloadJson TEXT NOT NULL, + AttemptCount INTEGER NOT NULL DEFAULT 0, + DeadLettered INTEGER NOT NULL DEFAULT 0 + ); + """); + + foreach (var row in rows) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + INSERT INTO Queue (RowId, AlarmId, EnqueuedUtc, PayloadJson) + VALUES ($id, $alarm, '2026-07-20T00:00:00.0000000Z', $payload) + """; + cmd.Parameters.AddWithValue("$id", row.RowId); + cmd.Parameters.AddWithValue("$alarm", row.AlarmId); + cmd.Parameters.AddWithValue("$payload", row.Payload); + cmd.ExecuteNonQuery(); + } + } + + /// A file with a Queue table this build cannot read — no payload column. + private void SeedUnrecognisedLegacy() + { + using var conn = Open(_legacyPath); + Exec(conn, "CREATE TABLE Queue (RowId INTEGER PRIMARY KEY AUTOINCREMENT, Something TEXT);"); + Exec(conn, "INSERT INTO Queue (Something) VALUES ('x');"); + } + + private static SqliteConnection Open(string path) + { + var conn = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = path }.ToString()); + conn.Open(); + return conn; + } + + private static void Exec(SqliteConnection conn, string sql) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + cmd.ExecuteNonQuery(); + } + + public void Dispose() + { + _provider?.Dispose(); + SqliteConnection.ClearAllPools(); + + foreach (var baseName in new[] { _dbPath, _legacyPath, _legacyPath + ".migrated" }) + { + foreach (var suffix in new[] { "", "-wal", "-shm" }) + { + try { File.Delete(baseName + suffix); } catch { /* best effort */ } + } + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/LocalDbPairHarness.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/LocalDbPairHarness.cs index a221dee1..abfd7e75 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/LocalDbPairHarness.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/LocalDbPairHarness.cs @@ -265,7 +265,7 @@ public sealed class LocalDbPairHarness : IAsyncDisposable .Build(); return new ServiceCollection() - .AddZbLocalDb(config, LocalDbSetup.OnReady) + .AddZbLocalDb(config, db => LocalDbSetup.OnReady(db, config)) .BuildServiceProvider(); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs index 77481b9b..3e45a17d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs @@ -39,7 +39,7 @@ public sealed class LocalDbSetupTests : IDisposable .Build(); _provider = new ServiceCollection() - .AddZbLocalDb(configuration, LocalDbSetup.OnReady) + .AddZbLocalDb(configuration, db => LocalDbSetup.OnReady(db, configuration)) .BuildServiceProvider(); return _provider.GetRequiredService(); From 271fcc4e15ecf520607d67fedfc3915479adca88 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 04:39:32 -0400 Subject: [PATCH 05/13] test(localdb): alarm S&F convergence scenarios Two driver nodes replicating the alarm buffer over the real loopback h2c transport, through the real fail-closed interceptor. This is the test the phase exists to pass: schema, registration, sink rewire and drain gate can all be individually green while a pair still fails to share the undelivered alarm history that is the whole point of moving the queue. Rows are written through the production sink rather than hand-rolled INSERTs, so the id derivation, column set and delete-on-ack semantics under test are the ones production uses. The scenarios cover a buffered burst converging with the oplog draining to zero, delivered rows being removed from BOTH nodes as tombstones (the no-redeliver-after-failover property), writes during a transport outage surviving the rejoin, and the same event accepted on both nodes collapsing to one row. The positive control found a weak assertion. With RegisterReplicated("alarm_sf_events") commented out, three of the four went red immediately -- but the same-event-on-both-nodes scenario still PASSED, because with replication off each node trivially holds its own single copy and "one row on each" is satisfied by two databases that never spoke. It now also enqueues a distinct event on B and requires both nodes to hold two rows, which cannot be satisfied without convergence; it goes red under the control like the others. Control restored, all four green. Also updates the second exact-set replicated-tables pin, in LocalDbWiringTests. The plan predicted these pins would go red here, and that is them working: one of them is asserted through the full driver DI graph rather than the OnReady callback, so it catches a registration that exists in the callback but never reaches a running host. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- ...7-20-localdb-adoption-phase2.md.tasks.json | 5 +- .../LocalDb/AlarmSfConvergenceTests.cs | 197 ++++++++++++++++++ .../LocalDbWiringTests.cs | 4 +- 3 files changed, 202 insertions(+), 4 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfConvergenceTests.cs diff --git a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json index 512b83cc..7e93a5bd 100644 --- a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json +++ b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json @@ -46,10 +46,11 @@ { "id": 5, "subject": "Task 5: Convergence + failover scenarios in the pair harness (+ positive control)", - "status": "pending", + "status": "completed", "blockedBy": [ 3 - ] + ], + "note": "4 scenarios green. POSITIVE CONTROL RUN (RegisterReplicated(alarm_sf_events) commented out): 3/4 went red immediately; the 4th (same-event-on-both-nodes) passed VACUOUSLY - with replication off each node trivially held its own single row. Strengthened by enqueuing a second distinct event on B and asserting BOTH nodes hold 2; it then went red under the control too. Control restored, all 4 green. Also fixed a SECOND exact-set pin the plan predicted: LocalDbWiringTests." }, { "id": 6, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfConvergenceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfConvergenceTests.cs new file mode 100644 index 00000000..2ebf485a --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmSfConvergenceTests.cs @@ -0,0 +1,197 @@ +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.LocalDb; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.LocalDb; + +/// +/// LocalDb Phase 2 — two driver nodes replicating the alarm store-and-forward buffer over a real +/// loopback h2c transport, through the real fail-closed interceptor. +/// +/// +/// +/// This is the test the phase exists to pass. Everything upstream — the schema, the +/// registration, the sink rewire, the drain gate — can be individually green while a +/// redundant pair still fails to share the undelivered alarm history that is the whole +/// point of moving the queue here. +/// +/// +/// Rows are written through the production rather +/// than by hand-rolled INSERTs. The id derivation, the column set and the delete-on-ack +/// semantics under test are then the ones production actually uses. +/// +/// +[Collection("LocalDbPairConvergence")] +public sealed class AlarmSfConvergenceTests +{ + private static readonly Serilog.ILogger Log = new LoggerConfiguration().CreateLogger(); + + /// A writer that acknowledges everything — the drain's happy path. + private sealed class AckWriter : IAlarmHistorianWriter + { + public Task> WriteBatchAsync( + IReadOnlyList batch, CancellationToken cancellationToken) => + Task.FromResult>( + batch.Select(_ => HistorianWriteOutcome.Ack).ToList()); + } + + /// A writer that never acknowledges — a historian outage, which is when the buffer matters. + private sealed class OutageWriter : IAlarmHistorianWriter + { + public Task> WriteBatchAsync( + IReadOnlyList batch, CancellationToken cancellationToken) => + Task.FromResult>( + batch.Select(_ => HistorianWriteOutcome.RetryPlease).ToList()); + } + + private static AlarmHistorianEvent Event(string alarmId, int second) => new( + AlarmId: alarmId, + EquipmentPath: "line-1/eq-1", + AlarmName: "temp-high", + AlarmTypeName: "LimitAlarm", + Severity: AlarmSeverity.High, + EventKind: "Activated", + Message: "temperature high", + User: "system", + Comment: null, + TimestampUtc: new DateTime(2026, 7, 21, 0, 0, second, DateTimeKind.Utc)); + + /// The set of row ids on a node, ordered — equal sets on both nodes means converged. + private static async Task IdsAsync(ILocalDb db) + { + var rows = await db.QueryAsync( + "SELECT id FROM alarm_sf_events ORDER BY id", + static r => r.GetString(0)); + return string.Join(",", rows); + } + + private static async Task CountAsync(ILocalDb db) + { + var rows = await db.QueryAsync( + "SELECT COUNT(*) FROM alarm_sf_events", static r => r.GetInt64(0)); + return rows[0]; + } + + [Fact] + public async Task AlarmBurstOnA_ConvergesToB_AndOplogDrains() + { + await using var pair = new LocalDbPairHarness(); + await pair.StartAsync(); + + // The Primary's sink. Its drain is left unstarted so the burst stays queued and observable; + // an outage is exactly the state in which this buffer has to survive a node loss. + using var sink = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log); + for (var i = 0; i < 25; i++) + await sink.EnqueueAsync(Event($"eq/alarm-{i}", i), TestContext.Current.CancellationToken); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.B) == 25, + "node B to hold all 25 buffered alarm events"); + + (await IdsAsync(pair.B)).ShouldBe(await IdsAsync(pair.A)); + + // Oplog drains to empty once B acks: the steady state of a converged pair, and the proof + // the rows were genuinely acknowledged rather than merely sent. + await LocalDbPairHarness.WaitUntilAsync( + async () => await LocalDbPairHarness.OplogDepthAsync(pair.A) == 0, + "node A's oplog to drain after node B acknowledges the burst"); + } + + [Fact] + public async Task DeliveredRowsAreRemovedFromBothNodes() + { + // The no-redeliver-after-failover property, asserted at the data layer. An acknowledged row + // is DELETEd, and the delete has to reach the peer as a tombstone — otherwise a promoted + // standby would find the Primary's already-delivered events still queued and send them again. + await using var pair = new LocalDbPairHarness(); + await pair.StartAsync(); + + using var buffering = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log); + for (var i = 0; i < 5; i++) + await buffering.EnqueueAsync(Event($"eq/alarm-{i}", i), TestContext.Current.CancellationToken); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.B) == 5, + "node B to receive the buffered events"); + + // The historian comes back and the Primary drains. + using var draining = new LocalDbStoreAndForwardSink(pair.A, new AckWriter(), Log); + await draining.DrainOnceAsync(TestContext.Current.CancellationToken); + (await CountAsync(pair.A)).ShouldBe(0); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.B) == 0, + "node B to drop its copies once node A delivered them"); + + (await LocalDbPairHarness.TombstoneCountAsync(pair.B, "alarm_sf_events")) + .ShouldBe(5, "the deletes must arrive as tombstones, not vanish silently"); + } + + [Fact] + public async Task WritesWhileTransportDown_SurviveRejoin() + { + // The realistic outage shape: alarms keep arriving on the Primary while the pair link is + // down. Nothing may be lost when it comes back. + await using var pair = new LocalDbPairHarness(); + await pair.StartAsync(); + + using var sink = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log); + await sink.EnqueueAsync(Event("eq/before", 1), TestContext.Current.CancellationToken); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.B) == 1, + "the pre-outage event to reach node B"); + + await pair.StopPassiveAsync(); + + for (var i = 0; i < 10; i++) + await sink.EnqueueAsync(Event($"eq/during-{i}", 10 + i), TestContext.Current.CancellationToken); + + await pair.RestartPairAsync(); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.B) == 11, + "every event buffered during the outage to reach node B after the rejoin"); + + (await IdsAsync(pair.B)).ShouldBe(await IdsAsync(pair.A)); + } + + [Fact] + public async Task TheSameEventOnBothNodes_ConvergesToOneRow() + { + // Both adapters default-write while their redundancy role is unknown, so in the boot window + // the pair genuinely double-accepts the same fanned transition. Payload-derived ids are what + // turn that into one row instead of two; random GUIDs would leave the duplicate in the + // buffer for the eventual Primary to deliver twice. + await using var pair = new LocalDbPairHarness(); + await pair.StartAsync(); + + var shared = Event("eq/shared", 1); + using var sinkA = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log); + using var sinkB = new LocalDbStoreAndForwardSink(pair.B, new OutageWriter(), Log); + + await sinkA.EnqueueAsync(shared, TestContext.Current.CancellationToken); + await sinkB.EnqueueAsync(shared, TestContext.Current.CancellationToken); + + // A second, DISTINCT event on B. Without it the scenario would pass with replication + // switched off entirely — each node would hold its own single copy and "count == 1 on + // both" would be satisfied by two unconverged databases. + await sinkB.EnqueueAsync(Event("eq/only-on-b", 2), TestContext.Current.CancellationToken); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CountAsync(pair.A) == 2 && await CountAsync(pair.B) == 2, + "both nodes to hold two rows: the doubly-accepted event collapsed, plus B's own"); + + (await IdsAsync(pair.A)).ShouldBe(await IdsAsync(pair.B)); + + // Held, not merely reached: a late-arriving replica of the peer's copy of the shared event + // must not add a third row a moment later. + (await LocalDbPairHarness.HeldWithinAsync( + async () => await CountAsync(pair.A) > 2 || await CountAsync(pair.B) > 2, + TimeSpan.FromSeconds(3))) + .ShouldBeFalse("the duplicate must stay collapsed"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbWiringTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbWiringTests.cs index 77060548..82f6c36e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbWiringTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbWiringTests.cs @@ -84,7 +84,7 @@ public sealed class LocalDbWiringTests : IDisposable } [Fact] - public void DriverGraph_LocalDbResolvesAsASingletonWithBothCacheTablesRegistered() + public void DriverGraph_LocalDbResolvesAsASingletonWithEveryReplicatedTableRegistered() { var sp = BuildDriverGraph(); @@ -95,7 +95,7 @@ public sealed class LocalDbWiringTests : IDisposable // Exact set, both directions — a dropped registration replicates nothing; an extra one costs // three triggers and oplog volume on every write. first.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal) - .ShouldBe(["deployment_artifacts", "deployment_pointer"]); + .ShouldBe(["alarm_sf_events", "deployment_artifacts", "deployment_pointer"]); } [Fact] From f76d1f91e5d3401dce84d144d1379c58a4dbdefd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 04:43:37 -0400 Subject: [PATCH 06/13] docs+chore(localdb): phase-2 rig config + docs Enables the alarm store-and-forward sink on all four driver nodes of the docker-dev rig -- the replicating site-a pair and the default-OFF site-b pin -- so the live gate has a real buffer to watch converge, and can see that site-b's sink works as a plain node-local queue with no peer traffic. Two rig details worth stating rather than rediscovering. The endpoint is deliberately unresolvable: there is no HistorianGateway here, so every drain attempt fails, which is exactly the historian-outage state the buffer exists for. But it still has to be a syntactically valid absolute http(s) URI or the host refuses to start, because ServerHistorianOptionsValidator is consumer-gated -- AlarmHistorian:Enabled=true makes the endpoint required even while ServerHistorian:Enabled=false. And MaxAttempts is raised far above the production default of 10, which against a permanently unreachable gateway would dead-letter the entire queue about five minutes in, turning a buffering test into a dead-letter test. Docs record what an operator now has to know: that a rising queue depth on a Secondary is correct and on BOTH nodes is not (that shape is a redundancy snapshot naming neither node -- check node identity before suspecting the sink), that delivery is at-least-once across a failover by design with no dedup layer, and that AlarmHistorian:DatabasePath is removed but should be left in place through the upgrade because the migrator still reads it to find the file to copy. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- CLAUDE.md | 30 +++- docker-dev/docker-compose.yml | 42 ++++++ docs/AlarmHistorian.md | 141 +++++++++++++----- docs/AlarmTracking.md | 12 +- docs/Configuration.md | 5 +- docs/README.md | 2 +- docs/Redundancy.md | 8 +- .../2026-07-20-localdb-pair-replication.md | 75 ++++++++-- ...7-20-localdb-adoption-phase2.md.tasks.json | 5 +- 9 files changed, 258 insertions(+), 62 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 06c1f7e5..0bed5c12 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -219,10 +219,13 @@ The server supports configurable OPC UA transport security via the `OpcUa:Enable The server supports non-transparent warm/hot redundancy via the `Redundancy` section in `appsettings.json`. Two instances share the same Galaxy DB and the same mxaccessgw (under distinct `MxAccess.ClientName` values) but have unique `ApplicationUri` values. Each exposes `RedundancySupport`, `ServerUriArray`, and a dynamic `ServiceLevel` based on role and runtime health. The primary advertises a higher ServiceLevel than the secondary. See `docs/Redundancy.md` for the full guide. -## LocalDb pair-local config cache (Phase 1) +## LocalDb pair-local store (Phases 1 + 2) Every **driver-role** node keeps a consolidated `ZB.MOM.WW.LocalDb` SQLite database (the retired -LiteDB `LocalCache` subsystem was deleted as superseded). Phase-1 scope: it caches the +LiteDB `LocalCache` subsystem was deleted as superseded). It holds **three replicated tables**: +`deployment_artifacts` + `deployment_pointer` (Phase 1) and `alarm_sf_events` (Phase 2). + +**Phase 1** caches the **deployed-configuration artifact** (chunked, SHA-256-verified, newest-2 retention per cluster) so a driver node can **boot from cache when central SQL Server is unreachable** — `DriverHostActor` writes the cache after each successful apply (`IDeploymentArtifactCache` → `LocalDbDeploymentArtifactCache`, @@ -234,8 +237,27 @@ library's gRPC sync — **default-OFF, fail-closed bearer auth** (`LocalDbSyncAu on `LocalDb:SyncListenPort` (a dedicated Kestrel h2c listener) + `LocalDb:Replication:*`. On the docker-dev rig the **site-a pair** has replication on; central + site-b stay default-OFF (site-b is the pin). ApiKey via `${secret:}`/env, never committed (the rig's dev key is the committed-dev-secret -exception). See `docs/Configuration.md` (`LocalDb` section), `docs/Redundancy.md` (pair-local config -cache), and the runbook `docs/operations/2026-07-20-localdb-pair-replication.md`. +exception). + +**Phase 2** moved the **alarm store-and-forward buffer** off its standalone `alarm-historian.db` +and into the same database as `alarm_sf_events`, so a node that dies holding undelivered alarm +history no longer takes it to the grave. `SqliteStoreAndForwardSink` became +`LocalDbStoreAndForwardSink` (`Core.AlarmHistorian`), and the breaking config key +`AlarmHistorian:DatabasePath` was **removed** — it is still read, by `AlarmSfLegacyMigrator`, purely +to find a pre-consolidation file to copy across on first boot (renamed `.migrated` after the copy +commits). Because the buffer replicates, **only the node holding the Primary role drains it**: the +sink takes a `drainGate` delegate, Runtime supplies it from the `IRedundancyRoleView` singleton that +`DriverHostActor` publishes its `PrimaryGatePolicy` verdict to, and a gated-off node reports the new +`HistorianDrainState.NotPrimary`. Delivery is therefore **at-least-once across a failover, by +design** (no dedup layer). Row ids are a SHA-256 of the event payload rather than a GUID, so the +same event accepted on both nodes — which happens in every boot window, since +`HistorianAdapterActor` default-writes while its role is unknown — converges to one row. On the +docker-dev rig the site-a pair and site-b both run `AlarmHistorian__Enabled=true` against a +deliberately unresolvable gateway endpoint, so the buffer is always a live, non-draining queue. + +See `docs/Configuration.md` (`LocalDb` section), `docs/Redundancy.md` (pair-local config +cache), `docs/AlarmHistorian.md`, and the runbook +`docs/operations/2026-07-20-localdb-pair-replication.md`. ## LDAP Authentication diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index 2d2d712d..be3f3c59 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -328,6 +328,22 @@ services: # site-a-1 is the initiator: it binds the h2c sync listener on 9001 AND dials the peer. # Setting SyncListenPort makes Program.cs add a dedicated Http2-only Kestrel listener and # re-bind the primary HTTP port (an explicit Listen* otherwise discards ASPNETCORE_URLS). + # Alarm store-and-forward buffer, Phase 2. Enabled on BOTH halves of the pair so the + # buffer is a live, replicated table rather than an empty one — the live gate needs a real + # queue to watch converge, and needs to see that only the Primary drains it. + # + # There is no HistorianGateway on this rig, so the endpoint below is deliberately + # unresolvable: every drain attempt fails, which is exactly the historian-outage state the + # buffer exists for. Note the endpoint must still be a syntactically valid absolute http(s) + # URI or the host refuses to start — ServerHistorianOptionsValidator is consumer-gated, so + # AlarmHistorian:Enabled=true makes it required even while ServerHistorian:Enabled=false. + # + # MaxAttempts is raised far above the production default of 10. With a permanently + # unreachable gateway the default would dead-letter the whole queue about five minutes into + # the outage, turning a buffering test into a dead-letter test. + AlarmHistorian__Enabled: "true" + AlarmHistorian__MaxAttempts: "1000000" + ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" LocalDb__Path: "/app/data/otopcua-localdb.db" LocalDb__SyncListenPort: "9001" LocalDb__Replication__PeerAddress: "http://site-a-2:9001" @@ -365,6 +381,22 @@ services: # but does NOT dial (no PeerAddress). The replication stream is bidirectional, so a-1's # single dial carries both directions. Same key + MaxBatchSize as a-1 (byte-identical key # is mandatory — the interceptor fail-closes on a mismatch). + # Alarm store-and-forward buffer, Phase 2. Enabled on BOTH halves of the pair so the + # buffer is a live, replicated table rather than an empty one — the live gate needs a real + # queue to watch converge, and needs to see that only the Primary drains it. + # + # There is no HistorianGateway on this rig, so the endpoint below is deliberately + # unresolvable: every drain attempt fails, which is exactly the historian-outage state the + # buffer exists for. Note the endpoint must still be a syntactically valid absolute http(s) + # URI or the host refuses to start — ServerHistorianOptionsValidator is consumer-gated, so + # AlarmHistorian:Enabled=true makes it required even while ServerHistorian:Enabled=false. + # + # MaxAttempts is raised far above the production default of 10. With a permanently + # unreachable gateway the default would dead-letter the whole queue about five minutes into + # the outage, turning a buffering test into a dead-letter test. + AlarmHistorian__Enabled: "true" + AlarmHistorian__MaxAttempts: "1000000" + ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" LocalDb__Path: "/app/data/otopcua-localdb.db" LocalDb__SyncListenPort: "9001" LocalDb__Replication__ApiKey: "dev-site-a-localdb-sync-key" @@ -397,6 +429,11 @@ services: # site-b is the default-OFF pin: it gets the pair-local cache but NO SyncListenPort and NO # replication config, so no sync listener binds and ISyncStatus stays disconnected/Healthy. # This is what the live gate's check 8 verifies — the cache works locally with replication off. + # Alarm store-and-forward with replication OFF — the other half of the site-b pin. The sink + # must work as a plain node-local buffer here, with no sync listener and no peer traffic. + AlarmHistorian__Enabled: "true" + AlarmHistorian__MaxAttempts: "1000000" + ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" LocalDb__Path: "/app/data/otopcua-localdb.db" ports: - "4844:4840" @@ -422,6 +459,11 @@ services: Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" # site-b default-OFF pin (see site-b-1): cache on, replication off. + # Alarm store-and-forward with replication OFF — the other half of the site-b pin. The sink + # must work as a plain node-local buffer here, with no sync listener and no peer traffic. + AlarmHistorian__Enabled: "true" + AlarmHistorian__MaxAttempts: "1000000" + ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" LocalDb__Path: "/app/data/otopcua-localdb.db" ports: - "4845:4840" diff --git a/docs/AlarmHistorian.md b/docs/AlarmHistorian.md index 212afe4c..26dc8efd 100644 --- a/docs/AlarmHistorian.md +++ b/docs/AlarmHistorian.md @@ -42,7 +42,7 @@ unless noted. - **`NullAlarmHistorianSink`** — the no-op default for tests and deployments that don't historize alarms. It is the default DI binding (registered in the Runtime's `AddOtOpcUaRuntime`); production overrides it with - `SqliteStoreAndForwardSink`. + `LocalDbStoreAndForwardSink`. - **`AlarmHistorianEvent`** ([`AlarmHistorianEvent.cs`](../src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmHistorianEvent.cs)) — the source-agnostic event record: `AlarmId`, `EquipmentPath` (UNS path, @@ -61,42 +61,84 @@ unless noted. - **`HistorianSinkStatus`** — diagnostic snapshot surfaced to the AdminUI and `/healthz`: `QueueDepth`, `DeadLetterDepth`, `LastDrainUtc`, `LastSuccessUtc`, `LastError`, `DrainState`, and `EvictedCount`. -- **`HistorianDrainState`** — `Disabled` / `Idle` / `Draining` / `BackingOff`. +- **`HistorianDrainState`** — `Disabled` / `Idle` / `Draining` / `BackingOff` / + **`NotPrimary`** (ticking, but leaving the replicated queue for the node that + holds the Primary role). --- -## SqliteStoreAndForwardSink +## LocalDbStoreAndForwardSink -[`SqliteStoreAndForwardSink.cs`](../src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/SqliteStoreAndForwardSink.cs) -is the production `IAlarmHistorianSink`. Construction takes a SQLite database -path, an `IAlarmHistorianWriter`, a logger, and optional `batchSize` (default -100), `capacity` (default 1,000,000), `deadLetterRetention` (default 30 days), -and a test clock. +[`LocalDbStoreAndForwardSink.cs`](../src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs) +is the production `IAlarmHistorianSink`. Construction takes the node's +`ILocalDb`, an `IAlarmHistorianWriter`, a logger, and optional `batchSize` +(default 100), `capacity` (default 1,000,000), `deadLetterRetention` (default 30 +days), a test clock, and a **`drainGate`**. + +### The drain gate + +`drainGate` is a `Func` consulted at the top of every tick; `false` skips +the tick entirely, leaving the queue untouched and reporting +`DrainState = NotPrimary`. + +It exists because the queue **replicates**. The Secondary holds a full copy of +the Primary's rows, and an ungated drain there would re-deliver every event, +continuously — not just at a failover. Runtime supplies the gate from +`IRedundancyRoleView`, a singleton `DriverHostActor` publishes its +`PrimaryGatePolicy` verdict to on every redundancy snapshot, so the drain agrees +with the inbound-write and native-ack gates by construction. + +Two postures are deliberate: + +- **Unset (the default) means drain.** So does an unpublished view. A deployment + that runs no redundancy never publishes a role, and defaulting closed would + silently stop its alarm history forever. +- **A gate that throws means "not now", never permission.** Draining on a gate + fault would put a pair into dual delivery exactly when its redundancy signal is + sick. ### Queue table -The sink owns one SQLite table (created on construction, WAL journal mode): +The sink owns one table in the node's consolidated LocalDb — created and +registered for replication by `LocalDbSetup.OnReady`, before any consumer can +resolve the sink: ```sql -CREATE TABLE Queue ( - RowId INTEGER PRIMARY KEY AUTOINCREMENT, - AlarmId TEXT NOT NULL, - EnqueuedUtc TEXT NOT NULL, - PayloadJson TEXT NOT NULL, -- JSON-serialized AlarmHistorianEvent - AttemptCount INTEGER NOT NULL DEFAULT 0, - LastAttemptUtc TEXT NULL, - LastError TEXT NULL, - DeadLettered INTEGER NOT NULL DEFAULT 0 +CREATE TABLE alarm_sf_events ( + id TEXT NOT NULL PRIMARY KEY, -- SHA-256 of payload_json + alarm_id TEXT NOT NULL, + enqueued_at_utc TEXT NOT NULL, + payload_json TEXT NOT NULL, -- JSON-serialized AlarmHistorianEvent + attempt_count INTEGER NOT NULL DEFAULT 0, + last_attempt_utc TEXT NULL, + last_error TEXT NULL, + dead_lettered INTEGER NOT NULL DEFAULT 0 ); -CREATE INDEX IX_Queue_Drain ON Queue (DeadLettered, RowId); +CREATE INDEX ix_alarm_sf_events_drain + ON alarm_sf_events (dead_lettered, enqueued_at_utc, id); ``` -`EnqueueAsync` does a single `INSERT` on the hot path. To avoid a -`SELECT COUNT(*)` on every enqueue, the sink keeps an in-memory non-dead-lettered -row counter (seeded at startup, kept current by every mutation, and re-synced -from storage every 10,000 enqueues to defend against drift). SQLite writer -contention is handled via `PRAGMA busy_timeout=5000` + WAL so an enqueue/drain -collision waits out the file lock instead of failing fast. +**The primary key is a hash of the payload, not an autoincrement rowid.** Two +reasons. Replication converges last-writer-wins *per primary key*, so two nodes +independently allocating rowid 7 to different alarms would silently overwrite one +another. And an equal-payload key makes the same event arriving twice collapse +into one row — which happens for real, because `HistorianAdapterActor` +default-writes while its redundancy role is unknown and both nodes of a pair +therefore accept the same fanned transition in every boot window. Two genuinely +distinct events cannot collide: `AlarmHistorianEvent` carries a full-precision +timestamp alongside the alarm id, kind, message and user. + +Ordering follows from that: the drain reads in `enqueued_at_utc` order (with `id` +as a tiebreak so the ordering is total), because a hashed key carries no +insertion sequence. + +`EnqueueAsync` does a single `INSERT … ON CONFLICT DO NOTHING` on the hot path. +To avoid a `SELECT COUNT(*)` on every enqueue, the sink keeps an in-memory +non-dead-lettered row counter (seeded at startup, kept current by every mutation, +and re-synced from storage every 10,000 enqueues to defend against drift — now +doubly warranted, since replication applies the peer's rows straight into the +table where no local code path can observe them). Pragmas and connection +lifetime belong to LocalDb; the sink no longer manages either. ### Drain worker @@ -104,13 +146,17 @@ collision waits out the file lock instead of failing fast. `System.Threading.Timer`** (not started automatically — tests drive `DrainOnceAsync` deterministically). Each tick: +0. Consults the drain gate. A closed gate returns immediately, leaving the queue + untouched and `DrainState = NotPrimary`. 1. Purges aged dead-lettered rows past the retention window. -2. Reads up to `batchSize` non-dead-lettered rows in `RowId` order. +2. Reads up to `batchSize` non-dead-lettered rows in `enqueued_at_utc, id` order. 3. Rows with un-deserializable payloads are dead-lettered immediately (by their - own `RowId`) so they can't stall the queue head. + own `id`) so they can't stall the queue head. 4. The remaining batch is handed to `IAlarmHistorianWriter.WriteBatchAsync`, and - each outcome is applied in one transaction: `Ack` deletes the row, - `PermanentFail` flips its `DeadLettered` flag, `RetryPlease` bumps its attempt + each outcome is applied in one transaction: `Ack` deletes the row (the delete + replicates as a tombstone, so the peer drops its copy and a promoted standby + cannot re-send it), `PermanentFail` flips its `dead_lettered` flag, + `RetryPlease` bumps its attempt count and leaves it queued. A row whose `AttemptCount` has reached the configured **`MaxAttempts`** cap (default 10) is dead-lettered automatically on the next drain tick rather than retried — this breaks infinite retry loops for poison events whose @@ -131,7 +177,7 @@ an unobserved task exception. **The durability guarantee is bounded by `capacity` (default 1,000,000 rows).** When the non-dead-lettered queue reaches capacity, `EnqueueAsync` evicts the -oldest non-dead-lettered rows (oldest `RowId` first) to make room, logs a WARN, +oldest non-dead-lettered rows (oldest `enqueued_at_utc` first) to make room, logs a WARN, and increments `HistorianSinkStatus.EvictedCount`. Under a sustained historian outage, accepted alarm events can therefore be dropped before delivery. A non-zero `EvictedCount` is a data-loss signal that requires operator attention — @@ -140,7 +186,7 @@ it surfaces silent loss without log scraping. ### Dead-letter + operator recovery `PermanentFail` and corrupt-payload rows are retained in-place with -`DeadLettered = 1` for the retention window (default 30 days) so operators can +`dead_lettered = 1` for the retention window (default 30 days) so operators can inspect them before the sweeper purges them. `RetryDeadLettered()` is the operator action (from the AdminUI) that clears the dead-letter flag and attempt count on every dead-lettered row, returning them to the regular queue with a @@ -173,17 +219,28 @@ the `alerts` topic (future). The real sink is opt-in via the `AlarmHistorian` section of `appsettings.json`. When `Enabled` is `false` (the default), `AddAlarmHistorian` registers `NullAlarmHistorianSink` and the feature is dormant. When `Enabled` is `true`, -`AddAlarmHistorian` constructs `SqliteStoreAndForwardSink` and registers +`AddAlarmHistorian` constructs `LocalDbStoreAndForwardSink` and registers `GatewayAlarmHistorianWriter` as the `IAlarmHistorianWriter`. This section carries -**only** the `Enabled` gate + the SQLite store-and-forward knobs — the downstream +**only** the `Enabled` gate + the store-and-forward knobs — the downstream gateway connection (endpoint / key / TLS) is sourced from the `ServerHistorian` -section (see [Historian.md](Historian.md)). +section (see [Historian.md](Historian.md)), and the queue's storage location is the +node's consolidated `LocalDb:Path` database (see +[Configuration.md](Configuration.md)). + +> **Where the queue lives (changed).** The buffer is a table — `alarm_sf_events` — in the node's +> consolidated LocalDb, not a standalone `alarm-historian.db`. That is what lets it **replicate to +> the redundant pair peer**, so undelivered alarm history survives losing the node holding it. +> Two consequences follow, both covered in the +> [pair-replication runbook](operations/2026-07-20-localdb-pair-replication.md): +> **only the Primary drains** (the Secondary holds a full replica and would otherwise re-send +> everything), and **delivery is at-least-once across a failover** by design. On first boot after +> the upgrade, `AlarmSfLegacyMigrator` copies any existing `alarm-historian.db` across and renames +> it `.migrated`. ```json { "AlarmHistorian": { "Enabled": true, - "DatabasePath": "C:\\ProgramData\\OtOpcUa\\alarmhistorian.db", "BatchSize": 100, "DrainIntervalSeconds": 5, "Capacity": 1000000, @@ -194,8 +251,7 @@ section (see [Historian.md](Historian.md)). | Key | Type | Default | Description | |---|---|---|---| -| `Enabled` | bool | `false` | Enable the SQLite store-and-forward sink (drains to the HistorianGateway `SendEvent` path). `false` → `NullAlarmHistorianSink`. | -| `DatabasePath` | string | `alarm-historian.db` | Path to the SQLite queue file. Created on first use (WAL mode). Set an **absolute** path in production. | +| `Enabled` | bool | `false` | Enable the durable store-and-forward sink (drains to the HistorianGateway `SendEvent` path). `false` → `NullAlarmHistorianSink`. Requires LocalDb to be registered — an enabled sink with no `ILocalDb` fails at resolution rather than silently discarding events. | | `BatchSize` | int | `100` | Max rows per drain cycle handed to `IAlarmHistorianWriter.WriteBatchAsync`. | | `DrainIntervalSeconds` | int | `5` | Seconds between drain-worker ticks. | | `Capacity` | long | `1000000` | Max queued rows before the sink evicts the oldest (data-loss signal via `EvictedCount`). | @@ -207,6 +263,15 @@ section (see [Historian.md](Historian.md)). > `RuntimeDb:EventReadsEnabled=true`. The old Wonderware connection keys (`SharedSecret` / > `AlarmHistorian:Host`/`Port`/`UseTls`/`ServerCertThumbprint`) were pruned. +> **`DatabasePath` was removed.** The queue no longer owns a file. The key is still *read*, by +> `AlarmSfLegacyMigrator`, purely to locate a pre-consolidation `alarm-historian.db` to migrate — +> so leave it in place through the upgrade, then delete it once every node has an +> `alarm-historian.db.migrated` sidecar. A leftover value configures nothing. + +> **`DrainState` gained `NotPrimary`.** A gated-off Secondary reports it instead of `Idle`, so a +> rising queue depth there is distinguishable from a stalled drain. Both nodes reporting it means +> nobody is draining — see the runbook. + > Dev and docker-dev deployments leave `Enabled` unset (defaults to `false`) so alarm transitions historize to nowhere unless a HistorianGateway is configured. --- @@ -221,3 +286,5 @@ section (see [Historian.md](Historian.md)). - [ScriptedAlarms.md](ScriptedAlarms.md) — the scripted-alarm engine that emits most events into this sink. - [ServiceHosting.md](ServiceHosting.md) — the external HistorianGateway backend. +- [operations/2026-07-20-localdb-pair-replication.md](operations/2026-07-20-localdb-pair-replication.md) + — where the queue lives, the Primary-only drain, and the one-time legacy migration. diff --git a/docs/AlarmTracking.md b/docs/AlarmTracking.md index 4f00551d..399359b7 100644 --- a/docs/AlarmTracking.md +++ b/docs/AlarmTracking.md @@ -365,12 +365,16 @@ AB CIP ALMD) route to AVEVA Historian via the HistorianGateway: - `IAlarmHistorianSink` is the DI-registered intake contract. The default binding is `NullAlarmHistorianSink` (registered in `ServiceCollectionExtensions.AddOtOpcUaRuntime`). Production - deployments override it with `SqliteStoreAndForwardSink` wrapping + deployments override it with `LocalDbStoreAndForwardSink` wrapping `GatewayAlarmHistorianWriter` (the HistorianGateway `SendEvent` path) — see [ServiceHosting.md](ServiceHosting.md) for the HistorianGateway setup. -- `SqliteStoreAndForwardSink` queues each transition to a local - SQLite database and drains in the background via an - `IAlarmHistorianWriter`. **The durability guarantee is bounded**: the +- `LocalDbStoreAndForwardSink` queues each transition into the node's + consolidated LocalDb — where it **replicates to the redundant pair peer**, + so undelivered alarm history survives losing the node holding it — and + drains in the background via an `IAlarmHistorianWriter`. **Only the node + holding the Primary role drains**, since the peer holds a full replica; + delivery is consequently at-least-once across a failover, by design. See + [AlarmHistorian.md](AlarmHistorian.md). **The durability guarantee is bounded**: the queue capacity defaults to 1,000,000 rows; under a sustained historian outage, older non-dead-lettered rows are evicted (oldest first) to make room for new events. The `HistorianSinkStatus.EvictedCount` diff --git a/docs/Configuration.md b/docs/Configuration.md index 5f01bfdc..c4b1ecb3 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -136,8 +136,9 @@ The historian backend is the external **`ZB.MOM.WW.HistorianGateway`** sidecar, `ZB.MOM.WW.HistorianGateway.Client` gRPC package (the retired Wonderware TCP sidecar is documented at [`docs/drivers/Historian.Wonderware.md`](drivers/Historian.Wonderware.md)). The OtOpcUa host reads three appsettings sections — `ServerHistorian` (read path + gateway connection), `ContinuousHistorization` -(FasterLog outbox + recorder draining to `WriteLiveValues`), and `AlarmHistorian` (SQLite store-and-forward -alarm sink draining to `SendEvent`). The gateway connection (endpoint / key / TLS) lives **only** in +(FasterLog outbox + recorder draining to `WriteLiveValues`), and `AlarmHistorian` (the store-and-forward +alarm sink draining to `SendEvent`; its buffer lives in the node's `LocalDb:Path` database, not a +file of its own). The gateway connection (endpoint / key / TLS) lives **only** in `ServerHistorian`; the other two sections source it from there. The gateway API key is supplied via the environment variable **`ServerHistorian__ApiKey`** — never committed diff --git a/docs/README.md b/docs/README.md index c5df5b74..6c320064 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,7 +30,7 @@ The project was originally called **LmxOpcUa** (a single-driver Galaxy/MXAccess | [Subscriptions.md](v1/Subscriptions.md) | Monitored items → `ISubscribable` + per-driver subscription refcount (v1 archive) | | [AlarmTracking.md](AlarmTracking.md) | `IAlarmSource` + `AlarmSurfaceInvoker` + OPC UA alarm conditions — native Galaxy alarms end-to-end (live) | | [AlarmTracking.md](v1/AlarmTracking.md) | Original alarm-tracking write-up (v1 archive) | -| [AlarmHistorian.md](AlarmHistorian.md) | `Core.AlarmHistorian` store-and-forward SQLite sink — `SqliteStoreAndForwardSink`, `IAlarmHistorianWriter`, dead-letter/retry/eviction | +| [AlarmHistorian.md](AlarmHistorian.md) | `Core.AlarmHistorian` store-and-forward sink — `LocalDbStoreAndForwardSink`, the replicated `alarm_sf_events` buffer + Primary-only drain, `IAlarmHistorianWriter`, dead-letter/retry/eviction | | [DataTypeMapping.md](v1/DataTypeMapping.md) | Per-driver `DriverAttributeInfo` → OPC UA variable types (v1 archive — live mapping is in `src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/DataTypeMap.cs`) | | [IncrementalSync.md](IncrementalSync.md) | Address-space rebuild on redeploy + `sp_ComputeGenerationDiff` | | [HistoricalDataAccess.md](v1/HistoricalDataAccess.md) | `IHistoryProvider` as a per-driver optional capability (v1 archive) | diff --git a/docs/Redundancy.md b/docs/Redundancy.md index 6692bed9..0a0dff1d 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -221,15 +221,19 @@ Under warm/hot redundancy both cluster nodes run `ScriptedAlarmHostActor` and ev - **`alerts` topic emission** — `ScriptedAlarmHostActor` and `DriverHostActor.ForwardNativeAlarm` subscribe to the `redundancy-state` DPS topic and cache the local node's `RedundancyRole`, then gate the cluster `alerts` publish through `PrimaryGatePolicy` (table above). The OPC UA condition-node write and inbound ack/shelve command processing remain **ungated** on both nodes so the secondary is always ready to serve clients after a failover. - **`HistorianAdapterActor` historization** — likewise Primary-gated so alarm historization is exactly-once across all alarm sources. The actor subscribes to the `alerts` DPS topic and translates each `AlarmTransitionEvent` → `AlarmHistorianEvent` before enqueuing it on the sink; scripted alarms therefore historize exactly once regardless of cluster size. +- **The alarm sink's DRAIN** — a second, independent gate on the same decision. The enqueue gate above is not sufficient any more: since Phase 2 the store-and-forward buffer lives in the replicated LocalDb, so the Secondary holds a full copy of the Primary's queued rows and an ungated drain there would re-deliver every event continuously. `LocalDbStoreAndForwardSink` takes a `drainGate` fed from `IRedundancyRoleView` — a singleton `DriverHostActor` publishes its `PrimaryGatePolicy` verdict to, because the drain runs on a timer and cannot receive `RedundancyStateChanged` itself. A gated-off node reports `HistorianDrainState.NotPrimary`. Delivery is **at-least-once across a failover** by design: rows the old Primary delivered just before dying may not have replicated their deletes yet. See [AlarmHistorian.md](AlarmHistorian.md). Net effect: each alarm transition appears **once** on `/alerts` and would historize once, not once per node. See [ScriptedAlarms.md](ScriptedAlarms.md) and [AlarmTracking.md](AlarmTracking.md) for the scripted-alarm engine internals. -## Pair-local config cache (LocalDb — Phase 1) +## Pair-local store (LocalDb — Phases 1 + 2) Independently of the ServiceLevel machinery above, every **driver-role** node keeps a consolidated -[`ZB.MOM.WW.LocalDb`](../CLAUDE.md) SQLite database that caches the **deployed-configuration +[`ZB.MOM.WW.LocalDb`](../CLAUDE.md) SQLite database. Phase 2 added the **alarm store-and-forward +buffer** (`alarm_sf_events`) to it alongside the config cache, so a node that dies holding +undelivered alarm history no longer takes it to the grave — see the Primary-only drain note above +and [AlarmHistorian.md](AlarmHistorian.md). Phase 1 caches the **deployed-configuration artifact** (chunked). Its job is a single failure mode redundancy does not otherwise cover: a driver node restarting into a **central-SQL-Server outage** would come up with no configuration at all. With the cache, it **boots from the last artifact it applied** instead, and logs a running-from-cache diff --git a/docs/operations/2026-07-20-localdb-pair-replication.md b/docs/operations/2026-07-20-localdb-pair-replication.md index 78422482..8b7fc525 100644 --- a/docs/operations/2026-07-20-localdb-pair-replication.md +++ b/docs/operations/2026-07-20-localdb-pair-replication.md @@ -1,18 +1,20 @@ # LocalDb pair replication — operations runbook -> **Phase 1 scope.** Every driver-role OtOpcUa node keeps a consolidated -> [`ZB.MOM.WW.LocalDb`](../../CLAUDE.md) SQLite database. Today it caches exactly one thing: the -> **deployed-configuration artifact**, chunked, so a node can **boot from cache when central SQL -> Server is unreachable**. That cache can *optionally* replicate to the node's redundant pair peer -> over the library's gRPC sync. **Replication is default-OFF and fail-closed.** This runbook covers -> enabling it, the operational rules that keep a pair converging, and the DB-inspection safety -> rules. +> **Scope.** Every driver-role OtOpcUa node keeps a consolidated +> [`ZB.MOM.WW.LocalDb`](../../CLAUDE.md) SQLite database. It holds two things: the +> **deployed-configuration artifact** (chunked, Phase 1) so a node can **boot from cache when +> central SQL Server is unreachable**, and the **alarm store-and-forward buffer** (Phase 2) so a +> node that dies holding undelivered alarm history does not take it to the grave. Both can +> *optionally* replicate to the node's redundant pair peer over the library's gRPC sync. +> **Replication is default-OFF and fail-closed.** This runbook covers enabling it, the operational +> rules that keep a pair converging, and the DB-inspection safety rules. ## What replicates, and what it buys you -- **Replicated tables:** `deployment_artifacts` (the artifact, split into base64 chunks) and - `deployment_pointer` (one current-deployment pointer per cluster). Registered — in this order, - after the DDL — by `LocalDbSetup.OnReady`. +- **Replicated tables:** `deployment_artifacts` (the artifact, split into base64 chunks), + `deployment_pointer` (one current-deployment pointer per cluster), and `alarm_sf_events` (the + alarm store-and-forward buffer). Registered — in this order, after the DDL — by + `LocalDbSetup.OnReady`. - **The payoff:** in a redundant driver pair, *either* node can be the one that applied the latest deploy and wrote the cache. With replication on, the peer holds a byte-identical copy. So a node that restarts into a central-SQL outage can boot the current config **even if it never applied @@ -22,6 +24,47 @@ deployed. Retention (newest-2 deployments per cluster) prunes on one node and the deletes replicate as tombstones. +## Alarm store-and-forward — what replication changes + +Phase 2 moved the alarm buffer out of its own `alarm-historian.db` and into this database. Three +consequences an operator needs to know: + +- **Only the Primary drains.** The buffer replicates, so the Secondary holds a full copy of the + Primary's queue. Its drain worker is gated on the same Primary decision the inbound-write and + native-ack gates use, and reports `DrainState = NotPrimary`. **A rising queue depth on a + Secondary is correct.** A rising queue depth on *both* nodes is not — see below. +- **Delivery is at-least-once across a failover, by design.** Rows the old Primary delivered + moments before it died may not have had their deletes replicated yet, so the new Primary will + re-send them. This is accepted, not a defect: a duplicate alarm-history row is recoverable, a + missing one is not. There is deliberately no dedup layer. +- **The buffer is bounded.** `AlarmHistorian:Capacity` (1,000,000 by default) still evicts the + oldest undelivered rows when the historian has been unreachable long enough. Watch + `EvictedCount` — non-zero means accepted alarm events were dropped before reaching the historian. + +### If BOTH nodes report `NotPrimary` + +Nobody is draining, and the queue is filling toward the capacity ceiling on both. The gate +default-DENIES while the redundancy role is unknown *and* a driver peer exists, so this is what a +redundancy snapshot that never names either node looks like — the identity-mismatch shape +`DriverHostActor` also warns about once ("redundancy snapshot omitted this node"). Check node +identity (`Cluster:PublicHostname` / `Cluster:Port` skew) before suspecting the sink. + +### One-time migration from `alarm-historian.db` + +On first boot after the upgrade, `AlarmSfLegacyMigrator` copies any pre-existing +`alarm-historian.db` into `alarm_sf_events` and renames the file to `alarm-historian.db.migrated`. + +- **Both nodes of a pair migrate independently**, and their files overlap — row ids are derived + from the event payload, so the same event on both nodes converges to one row rather than + duplicating. The new Primary drains the union. +- **The rename happens only after the copy commits.** A failure fails host startup with the legacy + file untouched; the `.migrated` sidecar is what makes a later boot a no-op. +- **A file whose shape is unrecognised is left alone** — not renamed, not copied — so it stays + available for inspection. +- The `AlarmHistorian:DatabasePath` key is **removed**. It is still *read* by the migrator, to + find the legacy file; it no longer configures anything. Delete it from appsettings once the + `.migrated` sidecar exists on every node. + ## Enabling replication on a pair Replication has two knobs, both under `LocalDb`: @@ -127,10 +170,22 @@ FROM __localdb_row_version WHERE table_name = 'deployment_pointer' ORDER BY pk_j -- Replication backlog (should drain to 0 when a pair is caught up) SELECT COUNT(*) FROM __localdb_oplog; + +-- Undelivered alarm history, and how much of it has given up +SELECT dead_lettered, COUNT(*) FROM alarm_sf_events GROUP BY dead_lettered; + +-- Why the dead-lettered rows died +SELECT alarm_id, attempt_count, last_attempt_utc, last_error +FROM alarm_sf_events WHERE dead_lettered = 1 ORDER BY last_attempt_utc DESC LIMIT 20; + +-- Convergence check for the buffer: identical dumps mean the peer holds the ORIGIN-stamped rows +SELECT pk_json, hlc, node_id, is_tombstone +FROM __localdb_row_version WHERE table_name = 'alarm_sf_events' ORDER BY pk_json; ``` ## See also - [`docs/Redundancy.md`](../Redundancy.md) — the pair-local config cache section. - [`docs/Configuration.md`](../Configuration.md) — the `LocalDb` appsettings section. +- [`docs/AlarmHistorian.md`](../AlarmHistorian.md) — the alarm sink, its knobs, and the drain. - The two-node convergence harness: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/`. diff --git a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json index 7e93a5bd..104b2e38 100644 --- a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json +++ b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json @@ -55,10 +55,11 @@ { "id": 6, "subject": "Task 6: Rig config + docs", - "status": "pending", + "status": "completed", "blockedBy": [ 3 - ] + ], + "note": "Rig: AlarmHistorian__Enabled=true on site-a-1/2 AND site-b-1/2, with MaxAttempts raised to 1e6 (D-4: the default 10 would dead-letter the queue ~5min into the rig's permanent gateway outage) and a deliberately unresolvable ServerHistorian__Endpoint. NOTE: the endpoint is REQUIRED - ServerHistorianOptionsValidator is consumer-gated on AlarmHistorian:Enabled, so an empty one fails host start. Docs: runbook, AlarmHistorian.md, AlarmTracking.md, Configuration.md, Redundancy.md, docs/README.md, CLAUDE.md." }, { "id": 7, From 4480a7d755f085e87201fbdbf214203215709d9f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 04:47:35 -0400 Subject: [PATCH 07/13] docs(localdb): record deviations D-6 and D-7 in the phase-2 recon D-6 (payload-hash migrator ids over mig-{node}-{legacyId}) and D-7 (the plan's Host.Tests project does not exist) were captured only in commit messages and the tasks file. They belong with the other deviations, where the next reader looks. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- docs/plans/2026-07-20-localdb-phase2-recon.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/plans/2026-07-20-localdb-phase2-recon.md b/docs/plans/2026-07-20-localdb-phase2-recon.md index f5714fba..e078a2c7 100644 --- a/docs/plans/2026-07-20-localdb-phase2-recon.md +++ b/docs/plans/2026-07-20-localdb-phase2-recon.md @@ -263,6 +263,30 @@ One genuine change: the drain's `ORDER BY` moves from `RowId ASC` to `enqueued_a a hashed TEXT id carries no insertion order. Round-trip ("O") timestamps sort lexicographically in chronological order, and `id` makes the ordering total; the index covers both. +### D-6 — The migrator keys on the payload hash, not `mig-{node}-{legacyId}` + +**The plan says** deterministic ids of the form `mig-{NodeName}-{legacyId}`, mirroring ScadaBridge. + +**Why that is not enough here.** Node-prefixing does solve the collision the legacy AUTOINCREMENT +key would otherwise cause — node A's `RowId 7` and node B's `RowId 7` are different alarms — but it +*preserves* a duplication that ought to be collapsed. A warm pair's two legacy files **overlap**: +`HistorianAdapterActor` default-writes while its redundancy role is unknown, so both nodes accepted +the same fanned transitions during every boot window they ever had. Prefixed ids would carry every +one of those duplicates into the merged buffer permanently, for the eventual Primary to deliver +twice. + +**The decision.** Reuse D-1's `AlarmSfSchema.DeriveId` (lifted out of the sink so both call sites +share one identity rule). Distinct events cannot collide; identical ones converge. It also removes +the migrator's only need for node identity from configuration, and makes a crash between commit and +rename harmless under `INSERT OR IGNORE` — which the plan's own idempotency requirement wanted +anyway. + +### D-7 — The migrator's tests live in `Host.IntegrationTests` + +The plan names `tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests`. **That project does not exist.** The +tests sit beside `LocalDbSetupTests` in `Host.IntegrationTests`, which is where every other +LocalDb-facing test already lives. They are offline — a temp file and no external services. + ### D-4 — The live gate needs `MaxAttempts` raised on the rig Not actionable until Task 8, recorded now so it is not rediscovered there. The rig has no From 2e4ccf7fe9c58c9ebda87efade88aca452672edf Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 05:03:57 -0400 Subject: [PATCH 08/13] chore(localdb): phase-2 DoD sweep Build: 0 errors solution-wide, and 0 warnings from every project this branch touches. The ~816 solution-wide warnings are pre-existing xUnit1051 / OTOPCUA0001 / CS86xx in untouched driver + client test projects. Tests: full solution run compared against a full run on a detached worktree at the pre-branch baseline 2e46d054. The two failure SETS are identical -- all 13 tests, same names, zero new failures. Net +26 tests: +3 Core.AlarmHistorian (drain gate), +6 Runtime (role view), +17 Host.IntegrationTests (migrator + convergence). Set comparison rather than counts, because the suite carries standing environment- and load-dependent failures a count would hide. The greps found real drift Task 6 missed -- eight live sites still naming the deleted SqliteStoreAndForwardSink, including the AdminUI /alarms/historian panel text, which is user-visible, and a in HistorianAdapterActor that resolved to nothing without warning. All repointed at LocalDbStoreAndForwardSink; CLAUDE.md's alarm-history paragraph now also records the LocalDb buffer and the primary-gated drain, and drops DatabasePath from the knob list. docs/AlarmTracking.md still promised an AlarmHistorianOptions.Validate() startup warning for a relative DatabasePath and an empty SharedSecret; both branches are gone, so it now says so. Code references to AlarmHistorian:DatabasePath reduce to exactly two intentional ones: AlarmSfLegacyMigrator.LegacyPathKey and its test. No `new SqliteConnection` remains anywhere in Core.AlarmHistorian. Recon doc gains the durable verification record: guard-deletion evidence for both vacuous passes, the two exact-set replicated-table pins (both assert set equality, so an added or a dropped registration fails), and the baseline test comparison with a per-failure account of why each of the 13 is not this branch's. Stops here per the plan. Task 8's live gate needs explicit go-ahead; nothing on this branch is to be merged. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- CLAUDE.md | 12 ++-- docs/AlarmTracking.md | 5 +- docs/drivers/Historian.Wonderware.md | 2 +- ...7-20-localdb-adoption-phase2.md.tasks.json | 5 +- docs/plans/2026-07-20-localdb-phase2-recon.md | 69 +++++++++++++++++++ .../GatewayAlarmHistorianWriter.cs | 2 +- ...wayHistorianServiceCollectionExtensions.cs | 2 +- .../Components/Pages/AlarmsHistorian.razor | 5 +- src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 2 +- .../Historian/HistorianAdapterActor.cs | 2 +- .../ServiceCollectionExtensions.cs | 2 +- .../GatewayAlarmWriterFactoryTests.cs | 2 +- 12 files changed, 92 insertions(+), 18 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0bed5c12..2e2db504 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -361,11 +361,13 @@ authorization uses the standard `AccessLevels.HistoryRead` bit set at materializ ### Alarm-history path (`AlarmHistorian` section) Alarm events are written through `GatewayAlarmHistorianWriter` (the gateway **`SendEvent`** path) behind -the durable **`SqliteStoreAndForwardSink`** — `AlarmHistorian:Enabled=true` swaps the `NullAlarmHistorianSink` -default for the SQLite store-and-forward queue, whose drain worker forwards batches to the gateway and uses -per-event outcomes to decide retry vs. dead-letter (never throws). The `AlarmHistorian` section carries -only the `Enabled` gate + the SQLite knobs (`DatabasePath`, `DrainIntervalSeconds`, `Capacity`, -`DeadLetterRetentionDays`, `BatchSize`, `MaxAttempts`) — the downstream gateway connection +the durable **`LocalDbStoreAndForwardSink`** — `AlarmHistorian:Enabled=true` swaps the `NullAlarmHistorianSink` +default for the store-and-forward queue, which buffers into the node's consolidated LocalDb as the +replicated `alarm_sf_events` table (see the LocalDb section) and whose drain worker forwards batches to +the gateway and uses per-event outcomes to decide retry vs. dead-letter (never throws). **Only the node +holding the Primary role drains** (a gated node reports `HistorianDrainState.NotPrimary`). The +`AlarmHistorian` section carries only the `Enabled` gate + the queue knobs (`DrainIntervalSeconds`, +`Capacity`, `DeadLetterRetentionDays`, `BatchSize`, `MaxAttempts`) — the downstream gateway connection (endpoint/key/TLS) is sourced from the `ServerHistorian` section. **Alarm-history `ReadEvents` requires the target gateway deployed with `RuntimeDb:EventReadsEnabled=true`** (the C2 SQL event-read workaround). diff --git a/docs/AlarmTracking.md b/docs/AlarmTracking.md index 399359b7..ce5e3c63 100644 --- a/docs/AlarmTracking.md +++ b/docs/AlarmTracking.md @@ -383,8 +383,9 @@ AB CIP ALMD) route to AVEVA Historian via the HistorianGateway: capacity, and dead-letter retention are tunable via the `AlarmHistorian` config section (`DrainIntervalSeconds`, `Capacity`, `DeadLetterRetentionDays`); `AlarmHistorianOptions.Validate()` logs a - startup warning for an empty `SharedSecret`, a relative `DatabasePath`, - or a non-positive knob. + startup warning for a non-positive knob. (It no longer warns about + `SharedSecret` or `DatabasePath` — both keys are gone: the connection + comes from `ServerHistorian`, and the queue lives in the LocalDb.) - `HistorianAdapterActor` (`src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs`) subscribes to the cluster `alerts` DPS topic, translates each diff --git a/docs/drivers/Historian.Wonderware.md b/docs/drivers/Historian.Wonderware.md index 839ca7fc..ebf18c0b 100644 --- a/docs/drivers/Historian.Wonderware.md +++ b/docs/drivers/Historian.Wonderware.md @@ -13,7 +13,7 @@ OtOpcUa now consumes the **`ZB.MOM.WW.HistorianGateway`** sidecar through the Gi - **HistoryRead** → `GatewayHistorianDataSource` over the `ServerHistorian` appsettings section. - **Alarm history** → `GatewayAlarmHistorianWriter` (the gateway `SendEvent` path) behind the durable - `SqliteStoreAndForwardSink`; alarm-history `ReadEvents` needs the gateway running + `LocalDbStoreAndForwardSink`; alarm-history `ReadEvents` needs the gateway running `RuntimeDb:EventReadsEnabled=true`. - **Continuous historization** → a crash-safe FasterLog outbox + `ContinuousHistorizationRecorder` draining to the gateway's `WriteLiveValues` (`ContinuousHistorization` section); needs the gateway diff --git a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json index 104b2e38..27c13a64 100644 --- a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json +++ b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json @@ -64,12 +64,13 @@ { "id": 7, "subject": "Task 7: DoD sweep (offline) \u2014 STOP and report after this", - "status": "pending", + "status": "completed", "blockedBy": [ 4, 5, 6 - ] + ], + "note": "Build: 0 errors solution-wide; 0 warnings from every project this branch touches (the ~816 solution-wide warnings are pre-existing xUnit1051/OTOPCUA0001/CS86xx in untouched driver + client test projects). GREPS FOUND REAL DRIFT Task 6 missed: 8 live sites still named the deleted SqliteStoreAndForwardSink - CLAUDE.md, the AdminUI /alarms/historian panel text (user-visible), HistorianAdapterActor (a that did NOT warn), Runtime + Host + 2 Driver.Historian.Gateway doc comments, 1 gateway test comment, docs/drivers/Historian.Wonderware.md; plus docs/AlarmTracking.md still claimed a Validate() startup warning for a relative DatabasePath (that branch is gone). All fixed. Code refs to AlarmHistorian:DatabasePath now reduce to exactly two intentional ones: AlarmSfLegacyMigrator.LegacyPathKey and its test. No 'new SqliteConnection' in Core.AlarmHistorian. Both exact-set pins assert the 3-table set, both directions. Guard-deletion evidence (both vacuous passes + the pin inventory) consolidated into the recon doc." }, { "id": 8, diff --git a/docs/plans/2026-07-20-localdb-phase2-recon.md b/docs/plans/2026-07-20-localdb-phase2-recon.md index e078a2c7..1d858f5a 100644 --- a/docs/plans/2026-07-20-localdb-phase2-recon.md +++ b/docs/plans/2026-07-20-localdb-phase2-recon.md @@ -300,3 +300,72 @@ Related: `Program.cs:170-173` shows an `AlarmHistorian:Enabled=true` / `ServerHistorian:Enabled=false` deployment is explicitly supported and warned about — which is exactly the rig's shape. Task 6 must confirm `ServerHistorianOptionsValidator` does not fail host start for a disabled section with an empty `Endpoint`. + +--- + +## Guard-deletion evidence (recorded at the Task 7 DoD sweep) + +Every guard this phase adds was proved by deleting it and watching the tests go red. Two of those +runs found tests that would have passed over a broken implementation. Both are recorded here because +the *shape* of the vacuity is reusable, not just the fix. + +### Control 1 — remove `IRedundancyRoleView.Publish` from `DriverHostActor` + +**3 of 6** `DriverHostActorRoleViewTests` went red. The other three assert +`ShouldServiceAsPrimary == true`, which is exactly the value the view is **seeded** with — so they +cannot distinguish "published true" from "never published at all". That split is expected and +acceptable *given* the three that do go red pin the false cases, which only a real publish can +produce. Worth knowing before someone reads a future 3/6 result as a regression. + +### Control 2 — remove `RegisterReplicated("alarm_sf_events")` from `LocalDbSetup.OnReady` + +**3 of 4** `AlarmSfConvergenceTests` went red immediately. The fourth, +`TheSameEventOnBothNodes_ConvergesToOneRow`, **passed vacuously**: with replication switched off +each node trivially held its own single row, and "count == 1 on both nodes" was satisfied by two +completely unconverged databases. The assertion could not tell convergence from isolation. + +**Fix:** enqueue a second, *distinct* event on node B and require **both** nodes to hold two rows. +Isolation now yields 1 and 2; only convergence yields 2 and 2. Re-run under the control: red. +Control restored: all 4 green. + +The general trap — an assertion whose expected value is also what a *disabled* system produces — +is the same one recorded in the `#485`/`#486` unreadable-artifact defect class. + +### Exact-set replicated-table pins + +Two, not one, and both assert set equality (ordered `ShouldBe` against the literal three-table list, +so an added *or* dropped registration fails): + +- `LocalDbSetupTests` — drives the production `OnReady` callback directly. +- `LocalDbWiringTests` — resolves `ILocalDb` from the full driver DI graph, which is what catches a + registration that never reaches a running host. + +## Test-suite evidence (Task 7 DoD sweep) + +Full solution run on the branch, compared against a **full solution run on a detached worktree at +the pre-branch baseline `2e46d054`**. Comparing failure *sets*, not counts, because the suite has a +standing set of environment- and load-dependent failures that a raw count would hide. + +**The two failure sets are identical — all 13 tests, same names, both runs. Zero new failures.** + +| Assembly | Baseline | Branch | Delta | +|---|---|---|---| +| `Core.AlarmHistorian.Tests` | 0 failed / 29 passed | 0 failed / **32** passed | **+3** (the drain-gate tests; 29 ported from the old sink's suite) | +| `Runtime.Tests` | 1 failed / 418 passed | 1 failed / **424** passed | **+6** (`DriverHostActorRoleViewTests`) | +| `Host.IntegrationTests` | 2 failed / 169 passed | 2 failed / **186** passed | **+17** (migrator + convergence + pin updates) | + +The 13 standing failures, and why each is not this branch's: + +- **3 × `DriverTypeNamesGuardTests`** — `ArgumentNullException (secretResolver)` out of + `GalaxyDriverFactoryExtensions.Register`, reached by reflection. A real pre-existing defect in a + guard test, reproduced exactly on the baseline worktree. Unrelated to this phase. +- **4 × `AbLegacy.IntegrationTests`**, **3 × `OpcUaClient.IntegrationTests`**, + **1 × `DriverProbeHandshakeE2eTests.AbCip_Green_AgainstSim`** — driver fixtures on the + `10.100.0.35` docker host are not running. +- **1 × `RoslynVirtualTagEvaluatorTests.Evaluate_racing_ClearCompiledScripts_never_fails_with_disposed`** + — times out under full-suite load; passes in isolation. +- **1 × `ContinuousHistorizationRecorderTests.Retry_after_writer_failure_eventually_acks`** — fails + under full-suite load ("writer must have been called at least twice"), and `Runtime.Tests` passes + **425/425** when run alone. Also fails on the baseline. Worth noting for anyone reading a future + run: this one is *newly observed* here only because the previous sweep's log was truncated, not + because the branch introduced it. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayAlarmHistorianWriter.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayAlarmHistorianWriter.cs index 02eb624a..1817c8a3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayAlarmHistorianWriter.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayAlarmHistorianWriter.cs @@ -9,7 +9,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway; /// /// backed by the HistorianGateway SendEvent path. The -/// drain worker behind SqliteStoreAndForwardSink calls +/// drain worker behind LocalDbStoreAndForwardSink calls /// and uses the returned per-event /// to decide retry vs. dead-letter, so this writer maps every /// gateway result — success ack, the published client's typed exception hierarchy, raw diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs index 6033bde7..4df51b33 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs @@ -47,7 +47,7 @@ public static class GatewayHistorian /// — the same single gateway the read path /// () targets. The Host's AddAlarmHistorian wiring supplies /// this as the concrete the durable - /// SqliteStoreAndForwardSink drain worker delegates to, sourcing the connection from the + /// LocalDbStoreAndForwardSink drain worker delegates to, sourcing the connection from the /// ServerHistorian section (endpoint/key/TLS) rather than the legacy Wonderware-shaped /// AlarmHistorian host/port. Resolves an and the writer's /// from , falling back to the null diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/AlarmsHistorian.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/AlarmsHistorian.razor index db03988c..a957ca51 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/AlarmsHistorian.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/AlarmsHistorian.razor @@ -18,8 +18,9 @@
Snapshot from the local node's HistorianAdapterActor. Default sink is a no-op (NullAlarmHistorianSink); production wires - SqliteStoreAndForwardSink draining to the HistorianGateway - (SendEvent) behind it. Polling every @PollSeconds s. + LocalDbStoreAndForwardSink — buffering into this node's LocalDb, and + draining to the HistorianGateway (SendEvent) only while this node holds + the Primary role. Polling every @PollSeconds s.
@if (_status is null) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index b9de794b..0da86839 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -136,7 +136,7 @@ if (hasDriver) // Config-gated durable alarm-historian sink. When the AlarmHistorian section is enabled this // overrides the NullAlarmHistorianSink default from AddOtOpcUaRuntime (last registration wins) - // with a SqliteStoreAndForwardSink draining to the gateway SendEvent writer. The alarm-write path + // with a LocalDbStoreAndForwardSink draining to the gateway SendEvent writer. The alarm-write path // targets the SAME single gateway as the read path, so its connection (endpoint/key/TLS) is sourced // from the ServerHistorian section. AlarmHistorianOptions supplies only the Enabled gate + the // SQLite store-and-forward knobs (consumed inside AddAlarmHistorian) — it carries no connection fields. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs index 93eb4142..af60b86b 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs @@ -15,7 +15,7 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian; /// Thin actor wrapper around . Engine code (ScriptedAlarmActor, /// Galaxy native alarm bridge, AB CIP ALMD reader) tells s to this /// actor; the actor enqueues them on the sink fire-and-forget. Production deployments register -/// against IAlarmHistorianSink; the sink owns the +/// against IAlarmHistorianSink; the sink owns the /// durable queue + drain-to-HistorianGateway-SendEvent loop. The actor here owns nothing operational beyond /// the message contract — its job is to keep the engine actors on Akka's mailbox without blocking /// them on disk I/O or gateway round-trips. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs index 7dc1e921..f00f9663 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs @@ -41,7 +41,7 @@ public static class ServiceCollectionExtensions /// /// Registers shared runtime services. Currently binds /// to as the default; production deployments - /// override this with SqliteStoreAndForwardSink wrapping the HistorianGateway alarm writer. + /// override this with LocalDbStoreAndForwardSink wrapping the HistorianGateway alarm writer. /// Call this BEFORE AddAkka. /// /// The service collection to register with. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayAlarmWriterFactoryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayAlarmWriterFactoryTests.cs index ddbfc80a..868ab7f0 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayAlarmWriterFactoryTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayAlarmWriterFactoryTests.cs @@ -7,7 +7,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests; /// /// Alarm-write cutover seam test (T13). The Host's AddAlarmHistorian wiring drains the durable -/// SqliteStoreAndForwardSink through this factory, so it must yield the gateway-backed writer — +/// LocalDbStoreAndForwardSink through this factory, so it must yield the gateway-backed writer — /// sourcing the single gateway's connection from (endpoint/key/TLS), /// not the legacy Wonderware-shaped AlarmHistorian host/port. Built offline: the underlying /// channel dials lazily, so both the factory and the writer ctor perform no network I/O (a bogus, From f9f1b8fceeb19cfa9168409dbdde3647315248d4 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 06:13:01 -0400 Subject: [PATCH 09/13] =?UTF-8?q?fix(localdb):=20phase-2=20live=20gate=20?= =?UTF-8?q?=E2=80=94=204=20production=20defects=20found=20and=20fixed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate record: docs/plans/2026-07-20-localdb-phase2-live-gate.md. Checks 1, 2, 5, 6 pass. Checks 3 and 4 are NOT satisfied — the defect they were meant to confirm turned out to be the opposite of what the plan assumed. Three defects crash-looped every driver node before check 1 could even run: 1. An empty ServerHistorian:ApiKey kills the host. ServerHistorianOptions- Validator exists to turn exactly that class of failure into a named OptionsValidationException, but its documented fail tier explicitly excluded ApiKey on the reasoning that a keyless client "degrades — the gateway rejects calls". It does not: the client validates its own options at construction, so the process dies during Akka startup and never makes a call. 2. UseTls disagreeing with the endpoint scheme kills the host too, in both directions (both messages confirmed in the shipped client assembly). Moving an endpoint from https to http without clearing UseTls is an ordinary migration slip. 3. Plaintext h2c was UNREACHABLE. HistorianGatewayClientAdapter forwarded the TLS-only options unconditionally, and AllowUntrustedServerCertificate defaults to false, so it always sent RequireCertificateValidation=true — which the client rejects outright when UseTls=false. Every http:// deployment crashed, though the scheme is documented as the supported way to select h2c, and the only workaround was to assert a certificate posture for a connection that has no certificate. The fourth was the blocker, and it is Phase 2's own: 4. The drain gate deferred to a Primary that cannot deliver. Redundancy roles are elected CLUSTER-WIDE; the alarm queue is PAIR-LOCAL. On the rig the elected driver Primary is central-1 — it carries the driver Akka role, replicates nobody's LocalDb and does not even run the alarm historian — so every driver node logged "Historian drain suspended", including the two site-b nodes that have no peer at all. Nothing drained anywhere, where before Phase 2 it drained fine. The cost is not a duplicate; it is the buffer growing to the capacity wall and evicting the audit trail it exists to protect. Fixed in three layers: a separate ShouldDrainAlarmHistory policy (unknown role drains; the two gates now deliberately disagree, and a test pins that); peer- host matching in DriverHostActor so a node stands down only for a Primary holding its rows; and AddAlarmHistorian short-circuiting the gate when replication is unconfigured — testing BOTH Replication:PeerAddress and SyncListenPort, since only the dialing half sets the former while both halves share the queue. Every one of these follows from the asymmetry: a false allow costs a duplicate row, which at-least-once delivery already accepts and payload-hash ids collapse; a false deny loses data silently. A third vacuous test, caught by the same delete-the-guard discipline: the role-view tests stayed green with the guard removed, because AwaitAssert polls until an assertion passes and the assertion was "reads open" — which is the SEEDED value, satisfied at the first poll before the actor processed anything. They now assert the sequence of published values through a recording view; the control then goes red for exactly the cases that matter. Migration evidence: 11 legacy rows across two deliberately overlapping files converged to exactly 9 identical rows on both nodes, proving D-6's payload-hash identity on real nodes rather than in a fixture. Open design fork, recorded in the gate doc rather than decided here: a pair cannot currently identify its own Primary, so both halves drain. Safe in every topology — nothing loses data — but the gate's de-duplication benefit is unrealised until roles are scoped per pair. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- docker-dev/docker-compose.yml | 36 +++++ ...7-20-localdb-adoption-phase2.md.tasks.json | 5 +- .../2026-07-20-localdb-phase2-live-gate.md | 151 ++++++++++++++++++ .../HistorianGatewayClientAdapter.cs | 17 +- .../ServerHistorianOptionsValidator.cs | 44 ++++- .../Drivers/DriverHostActor.cs | 46 +++++- .../Drivers/PrimaryGatePolicy.cs | 46 ++++++ .../Redundancy/IRedundancyRoleView.cs | 45 ++++-- .../ServiceCollectionExtensions.cs | 43 ++++- .../HistorianGatewayClientAdapterTests.cs | 55 +++++++ .../ServerHistorianOptionsValidatorTests.cs | 144 ++++++++++++++++- .../Drivers/DriverHostActorRoleViewTests.cs | 139 ++++++++++++---- .../Drivers/PrimaryGatePolicyTests.cs | 83 ++++++++++ .../AlarmHistorianRegistrationTests.cs | 104 ++++++++++++ 14 files changed, 885 insertions(+), 73 deletions(-) create mode 100644 docs/plans/2026-07-20-localdb-phase2-live-gate.md diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index be3f3c59..b6b33a7d 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -344,6 +344,15 @@ services: AlarmHistorian__Enabled: "true" AlarmHistorian__MaxAttempts: "1000000" ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" + # Required whenever the section is consumed: the gateway client validates its own options at + # construction and throws on an empty key, so a keyless node crash-loops during Akka startup + # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint + # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via + # ServerHistorian__ApiKey from the environment. + ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" + # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER + # direction, so an http endpoint with the default UseTls=true crash-loops the host. + ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" LocalDb__SyncListenPort: "9001" LocalDb__Replication__PeerAddress: "http://site-a-2:9001" @@ -397,6 +406,15 @@ services: AlarmHistorian__Enabled: "true" AlarmHistorian__MaxAttempts: "1000000" ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" + # Required whenever the section is consumed: the gateway client validates its own options at + # construction and throws on an empty key, so a keyless node crash-loops during Akka startup + # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint + # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via + # ServerHistorian__ApiKey from the environment. + ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" + # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER + # direction, so an http endpoint with the default UseTls=true crash-loops the host. + ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" LocalDb__SyncListenPort: "9001" LocalDb__Replication__ApiKey: "dev-site-a-localdb-sync-key" @@ -434,6 +452,15 @@ services: AlarmHistorian__Enabled: "true" AlarmHistorian__MaxAttempts: "1000000" ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" + # Required whenever the section is consumed: the gateway client validates its own options at + # construction and throws on an empty key, so a keyless node crash-loops during Akka startup + # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint + # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via + # ServerHistorian__ApiKey from the environment. + ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" + # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER + # direction, so an http endpoint with the default UseTls=true crash-loops the host. + ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" ports: - "4844:4840" @@ -464,6 +491,15 @@ services: AlarmHistorian__Enabled: "true" AlarmHistorian__MaxAttempts: "1000000" ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" + # Required whenever the section is consumed: the gateway client validates its own options at + # construction and throws on an empty key, so a keyless node crash-loops during Akka startup + # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint + # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via + # ServerHistorian__ApiKey from the environment. + ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" + # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER + # direction, so an http endpoint with the default UseTls=true crash-loops the host. + ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" ports: - "4845:4840" diff --git a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json index 27c13a64..2eb2139a 100644 --- a/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json +++ b/docs/plans/2026-07-20-localdb-adoption-phase2.md.tasks.json @@ -75,10 +75,11 @@ { "id": 8, "subject": "Task 8: Live gate on the docker-dev rig (needs explicit user go-ahead)", - "status": "pending", + "status": "completed", "blockedBy": [ 7 - ] + ], + "note": "Gate doc: docs/plans/2026-07-20-localdb-phase2-live-gate.md. Checks 1/2/5/6 PASS; checks 3+4 NOT SATISFIED (see below). FOUR production defects found + fixed, three of which crash-looped every driver node before check 1 could run: (1) empty ServerHistorian:ApiKey crashes the host - the validator had explicitly classified it as 'degrades', which is false because the gateway client validates its own options at construction; (2) UseTls vs endpoint scheme mismatch crashes likewise, both directions; (3) plaintext h2c was UNREACHABLE - the adapter forwarded TLS-only options unconditionally so every documented http:// deployment crashed; (4) THE BLOCKER - the drain gate deferred to a CLUSTER-WIDE elected Primary while the queue is PAIR-LOCAL. On the rig that Primary is central-1 (carries the driver Akka role, replicates nobody, runs no historian), so ALL FOUR driver nodes suspended their drains including unpaired site-b: silent permanent loss where pre-Phase-2 drained fine. Fixed in 3 layers (separate ShouldDrainAlarmHistory policy; peer-host matching in DriverHostActor; gate short-circuited entirely when replication is unconfigured, testing BOTH PeerAddress and SyncListenPort since only the dialer sets the former). Also fixed a THIRD vacuous test: AwaitAssert on the seeded-open value passed with the guard deleted; now asserts the sequence of PUBLISHED values via a recording view. Migration evidence: 11 legacy rows across two overlapping files -> exactly 9 identical rows on both nodes, proving D-6's payload-hash identity live. OPEN DESIGN FORK (in the gate doc): a pair cannot identify its own Primary, so both halves drain - safe (no loss, duplicates only) but the gate's de-dup benefit is unrealised." } ], "lastUpdated": "2026-07-21T00:00:00Z" diff --git a/docs/plans/2026-07-20-localdb-phase2-live-gate.md b/docs/plans/2026-07-20-localdb-phase2-live-gate.md new file mode 100644 index 00000000..8016bf18 --- /dev/null +++ b/docs/plans/2026-07-20-localdb-phase2-live-gate.md @@ -0,0 +1,151 @@ +# LocalDb Phase 2 — live gate (docker-dev rig) + +> Task 8 of `2026-07-20-localdb-adoption-phase2.md`. Evidence log. +> Rig: local `docker-dev/docker-compose.yml` — 2 central + site-a pair (replication ON) + site-b pair +> (default-OFF), one Akka cluster, one shared SQL. +> +> **Safety rules followed** (same as the Phase 1 gate): all DB inspection via `docker cp` of the +> `db`/`-wal`/`-shm` triplet out of the container, querying the copy — never host `sqlite3` on the +> live WAL file. + +## Headline + +**The gate did not get past its own first boot before finding real defects.** Four production bugs, +all fixed on-branch with regression tests, plus a fifth finding in the test suite itself. Three of +the four crash-looped every driver node; the fourth silently stopped alarm history everywhere. + +Checks 1, 2, 5 and 6 pass. **Checks 3 and 4 are NOT satisfied** — see "The blocker" below; they +cannot be run as written on a rig that has no per-pair redundancy roles, and the defect they were +meant to confirm turned out to be the opposite of what the plan assumed. + +## Seeding the migration + +The rig had no pre-consolidation `alarm-historian.db`, so one was synthesised per node and `docker +cp`-ed to `/app/alarm-historian.db` (the WORKDIR-relative default) into created-but-not-started +containers — the exact upgrade shape. + +Deliberately overlapping content, to test D-6 in production: **6 rows on site-a-1, 5 on site-a-2, +2 of them byte-identical payloads** (the warm-pair duplication `HistorianAdapterActor` produces in +every boot window). 11 legacy rows, **9 distinct**. + +## Defects found and fixed + +### 1. Empty `ServerHistorian:ApiKey` crash-loops the host (was classified "degrades") + +`AlarmHistorian:Enabled=true` requires the gateway alarm writer, which requires a key. +`HistorianGatewayClientOptions.Validate()` throws `The gateway API key must not be empty` inside the +client factory during Akka startup, so every driver node crash-looped. + +`ServerHistorianOptionsValidator` exists precisely to convert that class of failure into a named +`OptionsValidationException` — but its documented fail tier explicitly excluded `ApiKey`, reasoning +that a keyless client "degrades rather than crashes (the gateway rejects calls)". **That premise is +false**: the client validates its own options at construction, so the process dies before any call. +Fixed; the key is never echoed in the message. + +### 2. `UseTls` / endpoint-scheme mismatch crash-loops the host + +Immediately after fixing #1, the rig crash-looped again: `UseTls` defaults to `true`, the rig +endpoint is `http://`, and the client throws `UseTls requires an https gateway endpoint`. The +inverse (`https://` + `UseTls=false`) throws too — both strings confirmed in the shipped client +assembly. Setting an `http` endpoint without clearing `UseTls` is an ordinary migration slip that +takes the host down. Now validated in both directions. + +### 3. Plaintext h2c was *unreachable* — every `http://` deployment crashed + +Third crash-loop, and a genuine product bug rather than operator config. +`HistorianGatewayClientAdapter.Create` forwarded the TLS-only options unconditionally, and +`AllowUntrustedServerCertificate` defaults to `false`, so it always sent +`RequireCertificateValidation=true` — which the client rejects outright when `UseTls=false` +(`RequireCertificateValidation is a TLS-only option and requires UseTls=true`). + +CLAUDE.md and `docs/Historian.md` both document `http://` as the supported way to select h2c, but +**no h2c configuration could start**, and the only workaround was to claim +`AllowUntrustedServerCertificate=true` — a certificate posture for a connection with no certificate. +Fixed: TLS-only options stay at their defaults when `UseTls=false`. + +### 4. THE BLOCKER — the drain gate deferred to a Primary that cannot deliver + +With the rig finally up, **every driver node logged `Historian drain suspended`. Nothing drained +anywhere** — including the two site-b nodes, which have no redundancy peer and no replication at all. +Before Phase 2 this deployment drained fine, because the drain was ungated. + +Root cause, established from the rig rather than guessed: + +- `RedundancyStateActor` derives roles from Akka's **cluster-wide** `RoleLeader("driver")`. +- `central-1`/`central-2` carry **both** `admin` and `driver` Akka roles (`OTOPCUA_ROLES=admin,driver`). +- So the elected driver Primary is **central-1** — a node that replicates nobody's LocalDb and does + not even run the alarm historian. +- Every site node was `Secondary`, saw that a Primary existed, and stood down in its favour. + +**The redundancy role is a cluster-wide election; the alarm queue is pair-local.** Deferring on +"somebody is Primary" is therefore wrong in principle: in a multi-pair fleet the Primary is usually +in another pair and holds none of this node's rows. The consequence is not a duplicate — it is the +buffer growing on every node until the capacity wall evicts the oldest events, i.e. silent permanent +loss of the audit trail the queue exists to protect. + +Fixed in three layers, each with a positive control: + +1. `PrimaryGatePolicy.ShouldDrainAlarmHistory(role, queueSharingPeerIsPrimary)` — a **separate** + policy from `ShouldServiceAsPrimary`. Unknown role ⇒ drain. A node stands down only for a Primary + that holds its rows. The two gates now deliberately disagree, and a test pins that disagreement so + nobody "harmonises" them back. +2. `DriverHostActor` publishes that verdict, matching the snapshot's Primary against this node's + configured LocalDb replication peer host. +3. `AddAlarmHistorian` short-circuits the gate entirely when replication is not configured — a node + whose rows exist nowhere else must never defer. The predicate tests **both** `Replication:PeerAddress` + and `SyncListenPort`, because only the dialing half sets the former and both halves share the queue. + +The asymmetry of the error costs is what drives every one of these: a false allow costs a duplicate +history row, which the design already accepts (*at-least-once across a failover, by design*, and +payload-hash ids collapse identical events); a false deny loses data silently. + +### 5. A third vacuous test — `AwaitAssert` on the seeded value + +Deleting the guard (publishing the old write-gate verdict) left `DriverHostActorRoleViewTests` +**green**. `AwaitAssert` polls until an assertion passes, and the assertion was "the view reads +open" — which is also the value the view is *seeded* with, so it succeeded at the first poll, before +the actor had processed anything. It could not distinguish a correct publish from no publish at all. + +Fixed by asserting on the **sequence of published values** via a recording view, never on current +state. Re-run under the control: red, for exactly the cases that matter. This is the same +"absence-assertion passes by race" trap recorded for #485/#486, in a new disguise. + +## Checks + +| # | Check | Result | Evidence | +|---|---|---|---| +| 1 | Migration ran; `.migrated` sidecar on both site-a nodes; row counts match | ✅ PASS | `alarm-historian.db.migrated` present on both. 11 legacy rows across the two files → **exactly 9 rows on each node**, id sets byte-identical — the 2 shared payloads collapsed, proving D-6's payload-hash identity in production. Each node holds the *other's* rows (`legacy/a1-only-*` present on a-2 and vice versa). Legacy `attempt_count` distinction survived the copy (the 3-attempt row still leads the 0-attempt rows), `last_error` carried, 0 dead-lettered | +| 2 | Alarm burst converges: identical rowsets, oplog drains to 0, dead letters 0 | ✅ PASS | Identical id sets on both nodes; `__localdb_oplog` depth **0** on both; **0** dead-lettered. Drain visibly live against the deliberately unresolvable gateway — `attempt_count` climbing 33 → 42 → 168 with `last_error='retry-please'`, which is exactly the buffering behaviour the queue exists for | +| 3 | Only the primary drains | ❌ **NOT SATISFIED** — see defect 4 | As written the check assumes a per-pair Primary exists. On this rig the elected Primary is a central node outside both pairs, so the honest post-fix behaviour is that **all four nodes drain**. No node can currently identify a queue-sharing Primary: site-b is unpaired, and of the site-a pair only the dialing half knows its peer. Safe (no loss, duplicates only), but the check's intent is unproven | +| 4 | Failover: standby resumes draining; count double-delivered events | ⛔ **NOT RUN** | Superseded by defect 4. A meaningful failover test needs a rig where the pair itself elects a Primary; measuring "double delivery" is meaningless while both halves drain unconditionally | +| 5 | Both nodes restarted together: clean rejoin, identical counts, no I/O errors | ✅ PASS | `docker compose restart site-a-1 site-a-2` → **0** `disk I/O error` / `SQLITE_IOERR` / `corrupt` lines and 0 fatals in either log; both back to 9 rows / 0 dead / oplog 0, still identical | +| 6 | site-b default-OFF pin: sink works locally, no sync traffic | ✅ PASS | `alarm_sf_events` created and registered on both site-b nodes; port **9001 not bound** (`/proc/net/tcp`), no sync listener, no replication config; both drain their own queue (defect 4's third fix) with 0 fatals | + +## The open design question + +The gate leaves one question that is a design fork, not a bug to patch: + +**A pair cannot currently identify its own Primary.** Redundancy roles are elected cluster-wide, but +the queue is pair-local. Only the dialing half of a pair knows its peer (`Replication:PeerAddress`); +the listening half sets `SyncListenPort` only. So the drain gate can never fully engage, and both +halves of a replicated pair drain — safe, but duplicated. + +Three ways out, in increasing order of scope: + +- **(A) Configure both halves to dial.** Smallest change; makes the peer known on both sides so the + gate engages symmetrically. Needs confirmation that the LocalDb library accepts bidirectional dial, + and an operator-facing config change. +- **(B) Accept duplicates for unpaired-role deployments.** Change nothing further, document that the + gate only engages when redundancy roles are configured per pair. Cheapest; leaves the steady-state + double-delivery the gate was introduced to remove. +- **(C) Scope redundancy roles per pair in `RedundancyStateActor`.** Fixes the root cause for *every* + consumer of the role (ServiceLevel, scripted-alarm emit gate, inbound-write gate), all of which + inherit the same cluster-wide assumption. Largest blast radius, and the only one that makes + "Primary" mean what the docs say it means in a multi-pair fleet. + +The branch is safe in every topology as it stands — no configuration loses data — so this is a +question of when to reclaim the de-duplication benefit, not an outstanding hazard. + +## Not merged + +Per the plan, nothing on this branch is merged. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/HistorianGatewayClientAdapter.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/HistorianGatewayClientAdapter.cs index 5f429f30..c7a323d1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/HistorianGatewayClientAdapter.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/HistorianGatewayClientAdapter.cs @@ -57,16 +57,23 @@ public sealed class HistorianGatewayClientAdapter : IHistorianGatewayClient, IDi $"ServerHistorian:Endpoint must be an absolute http(s) URI (e.g. https://host:5222); got '{options.Endpoint}'."); } + // TLS-only options must stay at their defaults on a plaintext h2c connection: the client rejects + // each of them outright when UseTls=false (" is a TLS-only option and requires + // UseTls=true"). Forwarding them unconditionally made h2c UNREACHABLE — AllowUntrustedServer- + // Certificate defaults to false, so RequireCertificateValidation was always sent as true and + // every http:// deployment crashed at startup, even though the scheme is documented as the + // supported way to select h2c. There is no certificate to have a posture about here. var clientOptions = new HistorianGatewayClientOptions { Endpoint = endpointUri, ApiKey = options.ApiKey, UseTls = options.UseTls, - CaCertificatePath = options.CaCertificatePath, - // INVERTED mapping: ServerHistorianOptions.AllowUntrustedServerCertificate (opt-in to accept - // a self-signed cert) is the negation of the client's RequireCertificateValidation. Allowing - // an untrusted cert == not requiring validation; a pinned CaCertificatePath always verifies. - RequireCertificateValidation = !options.AllowUntrustedServerCertificate, + CaCertificatePath = options.UseTls ? options.CaCertificatePath : null, + // INVERTED mapping (TLS only): ServerHistorianOptions.AllowUntrustedServerCertificate + // (opt-in to accept a self-signed cert) is the negation of the client's + // RequireCertificateValidation. Allowing an untrusted cert == not requiring validation; a + // pinned CaCertificatePath always verifies. + RequireCertificateValidation = options.UseTls && !options.AllowUntrustedServerCertificate, DefaultCallTimeout = options.CallTimeout, LoggerFactory = loggerFactory, }; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs index ca727a5f..a9ebcc86 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs @@ -24,12 +24,25 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; /// in the Host, so Enabled covers it. /// /// -/// Fail tier = provably-crashing configs only. Only an empty / non-absolute / non-http(s) -/// Endpoint fails here (it throws in the factory). Empty ApiKey and non-positive -/// MaxTieClusterOverfetch degrade rather than crash (the gateway rejects calls / the node -/// manager surfaces a Bad read), so they stay operator warnings in +/// Fail tier = provably-crashing configs only. Three settings qualify, and they all fail +/// the same way — HistorianGatewayClientOptions.Validate() throws inside the client +/// factory before any call is attempted, so the host dies at startup: +/// an empty / non-absolute / non-http(s) Endpoint; an empty ApiKey ("The gateway +/// API key must not be empty"); and a UseTls that disagrees with the endpoint scheme +/// ("UseTls requires an https gateway endpoint" / "An https gateway endpoint requires UseTls"). +/// A non-positive MaxTieClusterOverfetch genuinely degrades rather than crashes (the node +/// manager surfaces a Bad read), so it stays an operator warning in /// . The Endpoint value is not a secret and -/// is echoed to make the error actionable; the ApiKey is never surfaced. +/// is echoed to make each error actionable; the ApiKey is never surfaced. +/// +/// +/// The ApiKey and UseTls checks were added after two consecutive live-rig +/// crash-loops (LocalDb Phase 2 gate). Both had been classified as degrading, on the +/// assumption that a bad client configuration would connect and be rejected by the gateway — but +/// the client validates its own options at construction, so the process dies during Akka startup +/// instead, which is precisely the failure this validator exists to convert into a named, +/// aggregated error. The lesson generalises: "degrades" is only true of settings the client +/// does not itself validate. /// /// public sealed class ServerHistorianOptionsValidator : OptionsValidatorBase @@ -62,5 +75,26 @@ public sealed class ServerHistorianOptionsValidator : OptionsValidatorBase private readonly IRedundancyRoleView? _redundancyRoleView; + /// + /// Host of this node's LocalDb replication partner — the only node that holds a copy of this + /// node's alarm queue, and therefore the only node it may ever stand down in favour of. + /// null when this node dials nobody (unpaired, or the listening half of a pair). + /// + private readonly string? _replicationPeerHost; + /// Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent /// identity mismatch (03/S5) would otherwise log on every snapshot. private bool _warnedSnapshotMissingLocalNode; @@ -372,7 +379,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers IDriverCapabilityInvokerFactory? invokerFactory = null, Func? driverMemberCountProvider = null, IDeploymentArtifactCache? deploymentArtifactCache = null, - IRedundancyRoleView? redundancyRoleView = null) => + IRedundancyRoleView? redundancyRoleView = null, + string? replicationPeerHost = null) => // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an // expression tree. Six IActorRef? parameters and several interface-typed ones mean a // mis-ordered argument is usually type-compatible and therefore compiles clean, then binds @@ -382,7 +390,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor, healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride, loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider, - deploymentArtifactCache, redundancyRoleView)); + deploymentArtifactCache, redundancyRoleView, replicationPeerHost)); /// Initializes a new DriverHostActor with the specified dependencies. /// Database context factory for configuration database access. @@ -433,10 +441,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers IDriverCapabilityInvokerFactory? invokerFactory = null, Func? driverMemberCountProvider = null, IDeploymentArtifactCache? deploymentArtifactCache = null, - IRedundancyRoleView? redundancyRoleView = null) + IRedundancyRoleView? redundancyRoleView = null, + string? replicationPeerHost = null) { _deploymentArtifactCache = deploymentArtifactCache; _redundancyRoleView = redundancyRoleView; + _replicationPeerHost = replicationPeerHost; _dbFactory = dbFactory; _localNode = localNode; _coordinatorOverride = coordinator; @@ -1465,6 +1475,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers } } + /// Host part of a host:port node id. + private static string HostOf(NodeId nodeId) + { + var text = nodeId.ToString(); + var colon = text.LastIndexOf(':'); + return colon < 0 ? text : text[..colon]; + } + /// The Primary-gate deny reason tag for the denial meter (secondary|detached|role-unknown). private string PrimaryGateDenyReason() => _localRole switch { @@ -1496,11 +1514,23 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers _localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId))); } - // Publish on EVERY snapshot, including ones that leave the cached role untouched: the other - // half of the decision is the driver member count, which moves with cluster membership and - // not with this message. Snapshots are re-published on a heartbeat, so a membership change - // is picked up within one heartbeat rather than waiting for a role change that may never come. - _redundancyRoleView?.Publish(ShouldServiceAsPrimary()); + // Publish on EVERY snapshot, including ones that leave the cached role untouched, so a role + // that changes without this node being named still reaches the drain within one heartbeat. + // + // NOTE the different policy: the drain gate keys on the role ALONE and opens when it is + // unknown, where the data-plane gate above resolves an unknown role by member count and + // denies. Publishing ShouldServiceAsPrimary() here suspended the drain on every node of a + // multi-driver cluster that had no redundancy configured — see ShouldDrainAlarmHistory. + // Defer ONLY to the node that holds a copy of our rows. The redundancy election is + // cluster-wide and the queue is pair-local, so "somebody is Primary" is not a licence to stop + // draining — that Primary is usually in another pair entirely, and cannot deliver our events. + var peerIsPrimary = + _replicationPeerHost is not null + && msg.Nodes.Any(n => n.Role == RedundancyRole.Primary + && HostOf(n.NodeId) == _replicationPeerHost); + + _redundancyRoleView?.Publish( + PrimaryGatePolicy.ShouldDrainAlarmHistory(_localRole, queueSharingPeerIsPrimary: peerIsPrimary)); } private void Stale() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/PrimaryGatePolicy.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/PrimaryGatePolicy.cs index 90486a19..86a0e3bf 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/PrimaryGatePolicy.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/PrimaryGatePolicy.cs @@ -25,4 +25,50 @@ public static class PrimaryGatePolicy RedundancyRole.Secondary or RedundancyRole.Detached => false, _ => driverMemberCount <= 1, // unknown: allow only when no driver peer exists }; + + /// + /// Decide whether this node should drain the replicated alarm store-and-forward queue. + /// + /// + /// + /// Deliberately not . That gate protects a shared + /// field device, where two nodes acting at once is dangerous and irreversible, so an unknown + /// role with a peer present must deny. This gate protects an append-only historian, and the + /// delivery contract is already at-least-once across a failover, by design — identical + /// events even collapse to a single row, because ids hash the payload. + /// + /// + /// The two error costs are therefore asymmetric: a false allow costs a duplicate history row; + /// a false deny silently stops the alarm audit trail and eventually evicts it at the capacity + /// wall. So this decision keys on the role alone and opens when the role is unknown — + /// no membership tie-break, because cluster-wide driver count says nothing about whether + /// this node has a redundant partner. + /// + /// + /// Found by the LocalDb Phase 2 live gate: on a rig of four driver nodes in one cluster with + /// no Redundancy section, borrowing the write gate's verdict suspended the drain on + /// every node — including unpaired ones — so nothing drained anywhere. + /// + /// + /// This node's last-known redundancy role, or null when unknown. + /// + /// Whether the node holding in the same snapshot is a node + /// that shares this node's queue — i.e. its LocalDb replication partner. + /// + /// Merely knowing that a Primary exists is not enough, because the redundancy role is + /// a CLUSTER-WIDE election while the alarm queue is PAIR-LOCAL. A fleet runs many pairs in + /// one cluster, so the elected driver Primary is usually in some other pair — and on the + /// docker-dev rig it is a central node, which carries the driver Akka role, replicates + /// nobody's LocalDb and does not even run the alarm historian. Deferring to it means this + /// node's events are delivered by no one, ever. + /// + /// + /// true to drain; false only when a node holding these same rows is doing it. + public static bool ShouldDrainAlarmHistory(RedundancyRole? localRole, bool queueSharingPeerIsPrimary) => + localRole switch + { + RedundancyRole.Primary => true, + RedundancyRole.Secondary or RedundancyRole.Detached => !queueSharingPeerIsPrimary, + _ => true, // unknown role ⇒ drain; a duplicate row beats a silent gap + }; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs index a74797f5..a9c41dff 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs @@ -3,44 +3,55 @@ using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; namespace ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; /// -/// A singleton snapshot of the Primary-gate decision, so code that lives outside an actor can -/// ask "should this node be servicing Primary-only work right now?". +/// A singleton snapshot of the alarm-history drain decision, so code that lives outside an actor +/// can ask "should this node be draining the alarm queue right now?". /// /// /// /// Exists for the alarm store-and-forward drain. That drain runs on a timer owned by the /// sink, not on an actor mailbox, so it cannot receive RedundancyStateChanged -/// directly — but it must be Primary-scoped, because the queue it drains replicates to the +/// directly — but it must be role-scoped, because the queue it drains replicates to the /// pair peer and two draining nodes would deliver every event twice. /// /// -/// Deliberately publishes the decision rather than the role and member count that -/// produced it, so stays the single place that decides. +/// Deliberately publishes the decision rather than the role that produced it, so +/// stays the single place that decides. +/// +/// +/// This is not the device-write gate. It is fed by +/// , which opens on an unknown role +/// where closes. The naming is +/// explicit about the consumer for that reason: an earlier revision published the +/// write-gate verdict here, and on a cluster with several driver nodes but no configured +/// redundancy it suspended the drain everywhere, so alarm history accumulated on every node +/// and left none. /// /// public interface IRedundancyRoleView { - /// Whether this node should service Primary-only work right now. - bool ShouldServiceAsPrimary { get; } + /// Whether this node should be draining the alarm store-and-forward queue right now. + bool ShouldDrainAlarmHistory { get; } /// Records a fresh decision. Called by DriverHostActor on every redundancy snapshot. - /// The decision produced. - void Publish(bool shouldServiceAsPrimary); + /// + /// The decision produced. + /// + void Publish(bool shouldDrainAlarmHistory); } /// Thread-safe backed by a volatile field. public sealed class RedundancyRoleView : IRedundancyRoleView { - // Seeded with the decision for "role unknown, no driver peer" rather than a bare false. An - // unpublished view and a node with no peer are genuinely the same situation, and they must - // behave the same: a deployment that runs no redundancy at all never publishes here, and - // defaulting closed would silently stop its alarm history forever. - private volatile bool _shouldServiceAsPrimary = - PrimaryGatePolicy.ShouldServiceAsPrimary(localRole: null, driverMemberCount: 0); + // Seeded with the decision for "role unknown" rather than a bare false. An unpublished view and a + // node whose role nothing ever reports are genuinely the same situation, and they must behave the + // same: a deployment that runs no redundancy at all never publishes here, and defaulting closed + // would silently stop its alarm history forever. + private volatile bool _shouldDrainAlarmHistory = + PrimaryGatePolicy.ShouldDrainAlarmHistory(localRole: null, queueSharingPeerIsPrimary: false); /// - public bool ShouldServiceAsPrimary => _shouldServiceAsPrimary; + public bool ShouldDrainAlarmHistory => _shouldDrainAlarmHistory; /// - public void Publish(bool shouldServiceAsPrimary) => _shouldServiceAsPrimary = shouldServiceAsPrimary; + public void Publish(bool shouldDrainAlarmHistory) => _shouldDrainAlarmHistory = shouldDrainAlarmHistory; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs index f00f9663..9cdbeddb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs @@ -107,6 +107,34 @@ public static class ServiceCollectionExtensions // posture for a deployment that runs no redundancy at all. services.TryAddSingleton(); + // THE GATE ONLY APPLIES TO A REPLICATED QUEUE. Standing down is only ever safe because some + // other node holds the same rows and will send them instead — and the only node that does is + // this node's LocalDb replication peer. Without replication configured, these rows exist here + // and nowhere else, so deferring to anyone means they are never delivered by anyone. + // + // This is not hypothetical. The redundancy role is a CLUSTER-WIDE election + // (RedundancyStateActor keys on Akka's RoleLeader("driver")), while the queue is PAIR-LOCAL. + // On the docker-dev rig the elected driver Primary is a central node — which carries the + // driver Akka role, replicates nobody's LocalDb, and does not even run the alarm historian — + // so every site node dutifully suspended its drain in favour of a node that could not + // possibly deliver its events. Scoping the gate to "is my queue actually shared?" is what + // keeps the two scopes from disagreeing. + // BOTH halves of a pair share the queue, but only one of them dials: the initiator sets + // Replication:PeerAddress, its partner only sets SyncListenPort and waits. Testing the dial + // side alone would leave the listening half permanently ungated — one drainer per pair by + // accident rather than by role, and the wrong one whenever the roles swap. + var replicated = + !string.IsNullOrWhiteSpace(configuration["LocalDb:Replication:PeerAddress"]) + || !string.IsNullOrWhiteSpace(configuration["LocalDb:SyncListenPort"]); + + if (!replicated) + { + Serilog.Log.Logger.ForContext().Information( + "Alarm historian: LocalDb replication is not configured, so this node's queue is not " + + "shared with any peer and the Primary drain gate does not apply — this node always " + + "drains its own alarm queue."); + } + services.AddSingleton(sp => { // LocalDbStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging). @@ -121,7 +149,7 @@ public static class ServiceCollectionExtensions capacity: opts.Capacity, deadLetterRetention: TimeSpan.FromDays(opts.DeadLetterRetentionDays), maxAttempts: opts.MaxAttempts, - drainGate: () => roleView.ShouldServiceAsPrimary); + drainGate: () => !replicated || roleView.ShouldDrainAlarmHistory); sink.StartDrainLoop(TimeSpan.FromSeconds(opts.DrainIntervalSeconds)); return sink; }); @@ -252,6 +280,16 @@ public static class ServiceCollectionExtensions // Registered by AddAlarmHistorian; absent when no durable sink is configured, in which // case there is nothing downstream to inform. var redundancyRoleView = resolver.GetService(); + // Host of this node's LocalDb replication partner: the ONLY node that holds a copy of this + // node's alarm queue, and so the only node it may stand down in favour of. Null when this + // node dials nobody, which correctly means "never stand down". + var replicationPeerHost = + Uri.TryCreate( + resolver.GetService()?["LocalDb:Replication:PeerAddress"], + UriKind.Absolute, + out var peerUri) + ? peerUri.Host + : null; // Root script logger backs the ScriptedAlarm host's engine + script logging. Registered in // Host DI inside the hasDriver block; may be absent in some role configs / test harnesses, // in which case the DriverHostActor gracefully skips spawning the ScriptedAlarm host. @@ -371,7 +409,8 @@ public static class ServiceCollectionExtensions scriptRootLogger: scriptRootLogger, invokerFactory: invokerFactory, deploymentArtifactCache: deploymentArtifactCache, - redundancyRoleView: redundancyRoleView), + redundancyRoleView: redundancyRoleView, + replicationPeerHost: replicationPeerHost), DriverHostActorName); registry.Register(driverHost); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/HistorianGatewayClientAdapterTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/HistorianGatewayClientAdapterTests.cs index 1a86a92e..55719278 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/HistorianGatewayClientAdapterTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/HistorianGatewayClientAdapterTests.cs @@ -22,6 +22,61 @@ public sealed class HistorianGatewayClientAdapterTests Assert.NotNull(adapter); } + /// + /// A plaintext h2c gateway (an http:// endpoint with UseTls=false) must + /// construct, using nothing but documented defaults. + /// + /// + /// + /// Found by the LocalDb Phase 2 live gate, on the third consecutive crash-loop of the rig. + /// docs/Historian.md and CLAUDE.md both document http:// as a supported + /// transport ("scheme selects transport: https = TLS, http = h2c"), but it was in fact + /// unreachable: this factory forwarded the TLS-only options unconditionally, and + /// defaults to + /// , so it always sent RequireCertificateValidation=true — + /// which the client rejects outright when UseTls=false + /// ("RequireCertificateValidation is a TLS-only option and requires UseTls=true"). + /// + /// + /// So every h2c deployment crashed at startup, and the only way to avoid it was to + /// set AllowUntrustedServerCertificate=true — asserting a certificate posture for a + /// connection that has no certificate at all. + /// + /// + [Fact] + public void Adapter_constructs_for_plaintext_h2c_endpoint() + { + var opts = new ServerHistorianOptions + { + Enabled = true, Endpoint = "http://localhost:5222", ApiKey = "histgw_x_y", UseTls = false, + }; + + using var adapter = HistorianGatewayClientAdapter.Create(opts, NullLoggerFactory.Instance); + + adapter.ShouldNotBeNull(); + } + + /// + /// A pinned CA path is likewise TLS-only, and must not leak into an h2c client — the same + /// rejection, reached by a different field. + /// + [Fact] + public void Adapter_ignores_tls_only_settings_on_a_plaintext_endpoint() + { + var opts = new ServerHistorianOptions + { + Enabled = true, + Endpoint = "http://localhost:5222", + ApiKey = "histgw_x_y", + UseTls = false, + CaCertificatePath = "/etc/ssl/certs/leftover-from-a-tls-config.pem", + }; + + using var adapter = HistorianGatewayClientAdapter.Create(opts, NullLoggerFactory.Instance); + + adapter.ShouldNotBeNull(); + } + /// /// archreview 06/S-11 — an enabled historian with an empty Endpoint must fail with a /// named, config-key-carrying (defense-in-depth for any diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ServerHistorianOptionsValidatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ServerHistorianOptionsValidatorTests.cs index d374c6b9..54baf356 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ServerHistorianOptionsValidatorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ServerHistorianOptionsValidatorTests.cs @@ -62,11 +62,18 @@ public sealed class ServerHistorianOptionsValidatorTests result.Failures.ShouldContain(f => f.Contains("ServerHistorian:Endpoint")); } - /// Enabled with a valid absolute https endpoint succeeds. + /// + /// Enabled with a valid absolute https endpoint and a key succeeds. The key is not + /// incidental: this case previously omitted it and still asserted success, which pinned a config + /// that actually crash-loops the host — see . + /// [Fact] public void Enabled_with_valid_https_endpoint_succeeds() { - var result = Sut().Validate(null, new ServerHistorianOptions { Enabled = true, Endpoint = "https://host:5222" }); + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://host:5222", ApiKey = "histgw_id_secret", + }); result.Succeeded.ShouldBeTrue(); } @@ -96,16 +103,145 @@ public sealed class ServerHistorianOptionsValidatorTests f.Contains("ServerHistorian:Endpoint") && f.Contains("AlarmHistorian:Enabled")); } - /// Alarm-only mode with a valid endpoint succeeds. + /// Alarm-only mode with a valid endpoint and a key succeeds. [Fact] public void Disabled_but_alarm_historian_enabled_with_valid_endpoint_succeeds() { var result = Sut(alarmHistorianEnabled: true) - .Validate(null, new ServerHistorianOptions { Enabled = false, Endpoint = "https://host:5222" }); + .Validate(null, new ServerHistorianOptions + { + Enabled = false, Endpoint = "https://host:5222", ApiKey = "histgw_id_secret", + }); result.Succeeded.ShouldBeTrue(); } + /// + /// An empty ApiKey on a consumed section fails, because it crashes — it does not + /// degrade. + /// + /// + /// + /// Found by the LocalDb Phase 2 live gate. The rig booted with + /// AlarmHistorian:Enabled=true and a valid endpoint but no key, and every driver + /// node crash-looped on + /// ArgumentException: The gateway API key must not be empty (Parameter 'ApiKey') + /// thrown by HistorianGatewayClientOptions.Validate() — reached through + /// GatewayHistorian.CreateAlarmWriter during Akka startup. + /// + /// + /// That is the *same* factory path, and the same crash-loop-under-a-restart-policy + /// failure mode, this validator was built to convert into a named + /// OptionsValidationException. The previous "empty ApiKey degrades, the gateway + /// just rejects calls" premise was simply wrong: the client validates its own options + /// at construction, so the process never reaches the point of making a call. + /// + /// + [Fact] + public void Consumed_with_empty_api_key_fails() + { + var result = Sut(alarmHistorianEnabled: true) + .Validate(null, new ServerHistorianOptions + { + Enabled = false, Endpoint = "https://host:5222", ApiKey = "", + }); + + result.Failed.ShouldBeTrue(); + result.Failures.ShouldContain(f => + f.Contains("ServerHistorian:ApiKey") && f.Contains("AlarmHistorian:Enabled")); + } + + /// The read path has the identical crash, so it is gated identically. + [Fact] + public void Enabled_with_empty_api_key_fails() + { + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://host:5222", ApiKey = "", + }); + + result.Failed.ShouldBeTrue(); + result.Failures.ShouldContain(f => f.Contains("ServerHistorian:ApiKey")); + } + + /// + /// The key must never be echoed. An endpoint is not a secret and is quoted to make the error + /// actionable; a key is, so the failure names only the setting. + /// + [Fact] + public void Api_key_failure_never_echoes_the_key() + { + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://host:5222", ApiKey = " ", + }); + + result.Failed.ShouldBeTrue(); + result.Failures.ShouldNotContain(f => f.Contains(" ", StringComparison.Ordinal)); + } + + /// A section nobody consumes may be keyless — no failure. + [Fact] + public void Disabled_with_empty_api_key_succeeds() + { + var result = Sut().Validate(null, new ServerHistorianOptions { Enabled = false, ApiKey = "" }); + + result.Succeeded.ShouldBeTrue(); + } + + /// + /// UseTls=true against an http:// endpoint fails — the third member of the + /// crash-at-construction family, and the easiest of the three for an operator to trip. + /// + /// + /// Also found by the Phase 2 live gate, immediately after the ApiKey fix: the rig points + /// at a deliberately unresolvable http:// endpoint, and UseTls defaults to + /// , so every node crash-looped a second time on + /// ArgumentException: UseTls requires an https gateway endpoint. Switching an endpoint + /// from https to http without clearing UseTls is an ordinary migration slip, and it takes + /// the host down rather than degrading it. + /// + [Fact] + public void Consumed_with_use_tls_against_http_endpoint_fails() + { + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "http://host:5222", ApiKey = "histgw_id_secret", UseTls = true, + }); + + result.Failed.ShouldBeTrue(); + result.Failures.ShouldContain(f => + f.Contains("ServerHistorian:UseTls") && f.Contains("http://host:5222")); + } + + /// An http endpoint with TLS explicitly off is the valid h2c shape — no failure. + [Fact] + public void Consumed_with_http_endpoint_and_tls_off_succeeds() + { + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "http://host:5222", ApiKey = "histgw_id_secret", UseTls = false, + }); + + result.Succeeded.ShouldBeTrue(); + } + + /// + /// The mirror slip — UseTls=false against an https:// endpoint — also fails, so the + /// scheme and the flag are pinned to agree in both directions rather than in one. + /// + [Fact] + public void Consumed_with_tls_off_against_https_endpoint_fails() + { + var result = Sut().Validate(null, new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://host:5222", ApiKey = "histgw_id_secret", UseTls = false, + }); + + result.Failed.ShouldBeTrue(); + result.Failures.ShouldContain(f => f.Contains("ServerHistorian:UseTls")); + } + /// /// Wiring guard (the "register-AND-consume" trap): resolving IOptions<ServerHistorianOptions>.Value /// after throws diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs index b5c3ed6b..c46952fa 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs @@ -21,10 +21,10 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; /// every alarm event twice. /// /// -/// These cases mirror 's matrix on purpose: -/// the view must reach exactly the same verdict as the inbound-write and native-ack gates, -/// because a second, subtly different notion of "am I the Primary" is how a pair ends up -/// with one node writing and neither draining. +/// These cases deliberately DIVERGE from in the +/// unknown-role case: the write gate denies when a peer may exist, this one allows. A duplicate +/// history row is cheap and already contemplated by the at-least-once contract; a silently +/// un-drained queue is not. See AlarmHistoryDrainGatePolicyTests. /// /// public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase @@ -43,15 +43,37 @@ public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase [Fact] public void An_unpublished_view_reads_as_primary() { - new RedundancyRoleView().ShouldServiceAsPrimary.ShouldBeTrue(); + new RedundancyRoleView().ShouldDrainAlarmHistory.ShouldBeTrue(); } + /// A Secondary closes the view — but only because the snapshot also names a Primary. [Fact] - public void Secondary_role_closes_the_view() + public void Secondary_role_closes_the_view_when_a_primary_exists() { var view = SpawnAndTellRole(RedundancyRole.Secondary, driverMemberCount: 2); - AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeFalse(), duration: Timeout); + AwaitAssert(() => view.Published.ShouldBe([false]), duration: Timeout); + } + + /// + /// A Secondary whose Primary is some OTHER pair's node keeps draining: that node holds none of + /// this node's rows, so deferring to it would deliver them never. + /// + [Fact] + public void Secondary_role_keeps_the_view_open_when_the_primary_is_not_its_peer() + { + var view = new RecordingRoleView(); + SpawnHost(view, driverMemberCount: 4).Tell(new RedundancyStateChanged( + [ + new NodeRedundancyState(TestNode, RedundancyRole.Secondary, + IsClusterLeader: false, IsRoleLeaderForDriver: false, AsOfUtc: DateTime.UtcNow), + // A Primary DOES exist — in another pair. It is not this node's replication peer. + new NodeRedundancyState(NodeId.Parse("some-other-pair:4053"), RedundancyRole.Primary, + IsClusterLeader: true, IsRoleLeaderForDriver: true, AsOfUtc: DateTime.UtcNow), + ], + CorrelationId.NewId())); + + AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout); } [Fact] @@ -59,67 +81,118 @@ public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase { var view = SpawnAndTellRole(RedundancyRole.Primary, driverMemberCount: 2); - AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeTrue(), duration: Timeout); + // Published, not current — `true` is the seed. See the note in the unknown-role theory. + AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout); } - /// Unknown role + a real driver peer ⇒ closed, matching the write gate's default-DENY. - [Fact] - public void Unknown_role_with_a_driver_peer_closes_the_view() + /// + /// Unknown role ⇒ the view stays open, even with driver peers present — the deliberate + /// divergence from the write gate's default-DENY. + /// + /// + /// + /// This case previously asserted the opposite, on the stated goal of mirroring the write + /// gate. The Phase 2 live gate showed that goal to be wrong for this consumer: the rig runs + /// four driver nodes in one cluster with no Redundancy section, so every role was + /// unknown and the driver count was 4 — and every node logged "Historian drain suspended", + /// including the two unpaired site-b nodes with no peer and no replication. Alarm history + /// drained nowhere, where before Phase 2 it drained fine. + /// + /// + /// See AlarmHistoryDrainGatePolicyTests for why the asymmetry of the two error costs + /// makes fail-open correct here and fail-closed correct for device writes. + /// + /// + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(4)] + public void Unknown_role_leaves_the_view_open_whatever_the_peer_count(int driverMemberCount) { // A snapshot that never names this node is how the role stays unknown in practice — the // documented identity-mismatch shape, not a contrived case. - var view = SpawnAndTellForeignSnapshot(driverMemberCount: 2); + var view = SpawnAndTellForeignSnapshot(driverMemberCount); - AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeFalse(), duration: Timeout); - } - - /// Unknown role + no driver peer ⇒ open, preserving the single-node posture. - [Fact] - public void Unknown_role_without_a_driver_peer_leaves_the_view_open() - { - var view = SpawnAndTellForeignSnapshot(driverMemberCount: 1); - - AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeTrue(), duration: Timeout); + // Assert on what was PUBLISHED, never on the current value. `true` is also the seeded + // default, so `AwaitAssert(() => view.ShouldDrainAlarmHistory.ShouldBeTrue())` passes at the + // first poll — before the actor has processed the snapshot — and therefore passes just as + // happily when the publish is wrong or absent. Verified: with the old write-gate verdict + // restored, the current-value form stayed green and only this form goes red. + AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout); } /// A demotion must move the view, not just an initial promotion. [Fact] public void A_demotion_closes_a_previously_open_view() { - var view = new RedundancyRoleView(); + var view = new RecordingRoleView(); var host = SpawnHost(view, driverMemberCount: 2); host.Tell(Snapshot(TestNode, RedundancyRole.Primary)); - AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeTrue(), duration: Timeout); + AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout); host.Tell(Snapshot(TestNode, RedundancyRole.Secondary)); - AwaitAssert(() => view.ShouldServiceAsPrimary.ShouldBeFalse(), duration: Timeout); + AwaitAssert(() => view.Published.ShouldBe([true, false]), duration: Timeout); } // ---------------- helpers ---------------- - private RedundancyRoleView SpawnAndTellRole(RedundancyRole role, int driverMemberCount) + /// + /// A view that remembers every published decision, so a test can assert on the actor's + /// action rather than on a state the seed already satisfies. + /// + private sealed class RecordingRoleView : IRedundancyRoleView { - var view = new RedundancyRoleView(); + private readonly List _published = []; + private readonly Lock _gate = new(); + private readonly RedundancyRoleView _inner = new(); + + /// Every value published so far, in order. + public IReadOnlyList Published + { + get { lock (_gate) return _published.ToArray(); } + } + + public bool ShouldDrainAlarmHistory => _inner.ShouldDrainAlarmHistory; + + public void Publish(bool shouldDrainAlarmHistory) + { + lock (_gate) _published.Add(shouldDrainAlarmHistory); + _inner.Publish(shouldDrainAlarmHistory); + } + } + + private RecordingRoleView SpawnAndTellRole(RedundancyRole role, int driverMemberCount) + { + var view = new RecordingRoleView(); SpawnHost(view, driverMemberCount).Tell(Snapshot(TestNode, role)); return view; } - private RedundancyRoleView SpawnAndTellForeignSnapshot(int driverMemberCount) + private RecordingRoleView SpawnAndTellForeignSnapshot(int driverMemberCount) { - var view = new RedundancyRoleView(); + var view = new RecordingRoleView(); SpawnHost(view, driverMemberCount).Tell(Snapshot(NodeId.Parse("some-other-node"), RedundancyRole.Primary)); return view; } + /// The peer this node replicates with — the only node it may ever stand down for. + private const string PeerHost = "the-peer"; + private IActorRef SpawnHost(IRedundancyRoleView view, int driverMemberCount) => Sys.ActorOf(DriverHostActor.Props( NewInMemoryDbFactory(), TestNode, coordinator: null, driverMemberCountProvider: () => driverMemberCount, - redundancyRoleView: view)); + redundancyRoleView: view, + replicationPeerHost: PeerHost)); + /// + /// A snapshot of a real pair: this node in , plus a peer holding the + /// other half. The peer matters — a Secondary only stands down when a Primary exists, so a + /// single-entry snapshot would silently change what these cases test. + /// private static RedundancyStateChanged Snapshot(NodeId node, RedundancyRole role) => new( [ @@ -127,6 +200,12 @@ public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase IsClusterLeader: role == RedundancyRole.Primary, IsRoleLeaderForDriver: role == RedundancyRole.Primary, AsOfUtc: DateTime.UtcNow), + new NodeRedundancyState( + NodeId.Parse($"{PeerHost}:4053"), + role == RedundancyRole.Primary ? RedundancyRole.Secondary : RedundancyRole.Primary, + IsClusterLeader: role != RedundancyRole.Primary, + IsRoleLeaderForDriver: role != RedundancyRole.Primary, + AsOfUtc: DateTime.UtcNow), ], CorrelationId.NewId()); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/PrimaryGatePolicyTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/PrimaryGatePolicyTests.cs index 07dac1d6..7a1262a9 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/PrimaryGatePolicyTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/PrimaryGatePolicyTests.cs @@ -36,3 +36,86 @@ public sealed class PrimaryGatePolicyTests public void Unknown_role_resolves_by_membership(int members, bool expected) => PrimaryGatePolicy.ShouldServiceAsPrimary(null, members).ShouldBe(expected); } + +/// +/// The alarm-history drain gate, which deliberately does not mirror the device-write gate. +/// +/// +/// +/// Why a second policy is correct here. The write gate protects a shared field device: +/// two nodes driving one PLC is dangerous and irreversible, so an unknown role with a peer +/// present must deny. The drain writes to an append-only historian instead, and Phase 2's +/// delivery contract is already at-least-once across a failover, by design — identical +/// events even collapse to one row, because ids are a hash of the payload. +/// +/// +/// So the two error costs are wildly asymmetric: a false allow costs a duplicate history row, +/// while a false deny silently stops the alarm audit trail and eventually evicts it at the +/// capacity wall. This gate therefore keys on the role alone and opens when the role is +/// unknown. +/// +/// +/// Found by the LocalDb Phase 2 live gate. The rig runs four driver nodes in one Akka +/// cluster with no Redundancy section, so every node's role was unknown and the +/// cluster-wide driver count was 4 — and every node, including the two unpaired site-b nodes +/// that have no peer and no replication at all, logged "Historian drain suspended". Nothing +/// drained anywhere. Before Phase 2 that deployment drained fine, because the drain was ungated. +/// +/// +public sealed class AlarmHistoryDrainGatePolicyTests +{ + [Theory] + // A Primary always drains its own queue. + [InlineData(RedundancyRole.Primary, true, true)] + [InlineData(RedundancyRole.Primary, false, true)] + // A Secondary stands down ONLY when a Primary actually exists to stand up. + [InlineData(RedundancyRole.Secondary, true, false)] + [InlineData(RedundancyRole.Detached, true, false)] + [InlineData(RedundancyRole.Secondary, false, true)] + [InlineData(RedundancyRole.Detached, false, true)] + public void A_known_role_stands_down_only_for_a_primary_that_exists( + RedundancyRole role, bool queueSharingPeerIsPrimary, bool expected) + => PrimaryGatePolicy.ShouldDrainAlarmHistory(role, queueSharingPeerIsPrimary).ShouldBe(expected); + + /// + /// Unknown role ⇒ drain, whoever else is out there. The first case the live gate broke on, and + /// the one that separates this policy from . + /// + [Theory] + [InlineData(true)] + [InlineData(false)] + public void An_unknown_role_drains(bool queueSharingPeerIsPrimary) + => PrimaryGatePolicy.ShouldDrainAlarmHistory(null, queueSharingPeerIsPrimary).ShouldBeTrue(); + + /// + /// A Secondary whose Primary is NOT its queue-sharing peer keeps draining — the deepest of the + /// live gate's findings. + /// + /// + /// RedundancyStateActor derives roles from Akka's cluster-wide RoleLeader("driver"), + /// but the alarm queue is pair-local. On the docker-dev rig the elected Primary is a CENTRAL node + /// — it carries the driver Akka role, replicates nobody's LocalDb, and does not even run the + /// alarm historian — so under a plain "a Primary exists ⇒ stand down" rule every site node + /// suspended its drain in favour of a node that could not possibly deliver its events. The + /// buffer then grows on every node until the capacity wall evicts the oldest: silent, permanent + /// loss of the audit trail the queue exists to protect. + /// + [Fact] + public void A_secondary_whose_peer_is_not_the_primary_keeps_draining() + { + // A Primary exists somewhere in the cluster, but it does not hold this node's rows. + PrimaryGatePolicy.ShouldDrainAlarmHistory(RedundancyRole.Secondary, queueSharingPeerIsPrimary: false) + .ShouldBeTrue("a Secondary must not defer to a Primary that cannot deliver its events"); + } + + /// + /// The divergence, pinned explicitly: with a peer present and no role known, the write gate + /// denies and the drain gate allows. If someone later "harmonises" the two, this fails. + /// + [Fact] + public void The_two_gates_deliberately_disagree_when_the_role_is_unknown_and_a_peer_exists() + { + PrimaryGatePolicy.ShouldServiceAsPrimary(localRole: null, driverMemberCount: 4).ShouldBeFalse(); + PrimaryGatePolicy.ShouldDrainAlarmHistory(localRole: null, queueSharingPeerIsPrimary: true).ShouldBeTrue(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs index 59d375e7..32531012 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/AlarmHistorianRegistrationTests.cs @@ -7,6 +7,7 @@ using Xunit; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; using ZB.MOM.WW.OtOpcUa.Runtime.Historian; +using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Historian; @@ -102,6 +103,109 @@ public sealed class AlarmHistorianRegistrationTests } } + /// + /// A node whose LocalDb is not replicated drains its own queue regardless of redundancy role. + /// + /// + /// + /// Standing down is only safe because a peer holds the same rows and will send them + /// instead. Without LocalDb:Replication:PeerAddress there is no such peer: the rows + /// are here and nowhere else, so any deferral means nobody ever delivers them. + /// + /// + /// Found by the Phase 2 live gate. The redundancy role is a CLUSTER-WIDE election while the + /// queue is PAIR-LOCAL, and on the docker-dev rig the elected driver Primary turned out to + /// be a central node — one that carries the driver Akka role, replicates nobody's LocalDb + /// and does not even run the alarm historian. Every site node suspended its drain in favour + /// of a node that could not possibly deliver its events. + /// + /// + [Theory] + // Neither key: an unpaired node. Its rows exist nowhere else. + [InlineData(null, null, true)] + // The dialing half of a pair — gated, because its partner holds the same rows. + [InlineData("http://peer:9001", null, false)] + // The LISTENING half. Same shared queue, but it never dials, so a PeerAddress-only test would + // wrongly class it as unpaired and leave it permanently ungated. + [InlineData(null, "9001", false)] + public async Task Only_an_unreplicated_node_ignores_the_role_view( + string? peerAddress, string? syncListenPort, bool expectDrain) + { + var tempDir = Path.Combine(Path.GetTempPath(), "otopcua-alarmhist-test-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + var dbPath = Path.Combine(tempDir, "node-local.db"); + + try + { + var config = ConfigFrom(new Dictionary + { + ["AlarmHistorian:Enabled"] = "true", + ["LocalDb:Path"] = dbPath, + ["LocalDb:Replication:PeerAddress"] = peerAddress, + ["LocalDb:SyncListenPort"] = syncListenPort, + }); + + var services = BaseServices(); + services.AddZbLocalDb(config, db => + { + using var connection = db.CreateConnection(); + AlarmSfSchema.Apply(connection); + db.RegisterReplicated(AlarmSfSchema.EventsTable); + }); + + // A role view pinned SHUT: this node is a Secondary and a Primary exists elsewhere. + services.AddSingleton(new StandDownRoleView()); + services.AddAlarmHistorian(config, (_, _) => new FakeWriter()); + + using (var provider = services.BuildServiceProvider()) + { + var sink = provider.GetRequiredService(); + await sink.EnqueueAsync(SampleEvent, CancellationToken.None); + + await ((LocalDbStoreAndForwardSink)sink).DrainOnceAsync(CancellationToken.None); + + var status = sink.GetStatus(); + if (expectDrain) + { + // Delivered and removed — not parked behind a gate that nobody else will open. + status.DrainState.ShouldNotBe(HistorianDrainState.NotPrimary); + status.QueueDepth.ShouldBe(0); + } + else + { + // Held for the peer that shares this queue. + status.DrainState.ShouldBe(HistorianDrainState.NotPrimary); + status.QueueDepth.ShouldBe(1); + } + } + } + finally + { + SqliteConnection.ClearAllPools(); + try { Directory.Delete(tempDir, recursive: true); } catch (IOException) { /* best effort */ } + } + } + + /// A view that always says "some other node is the Primary". + private sealed class StandDownRoleView : IRedundancyRoleView + { + public bool ShouldDrainAlarmHistory => false; + + public void Publish(bool shouldDrainAlarmHistory) { } + } + + private static AlarmHistorianEvent SampleEvent => new( + AlarmId: "eq/alarm-1", + EquipmentPath: "line-1/eq-1", + AlarmName: "temp-high", + AlarmTypeName: "LimitAlarm", + Severity: ZB.MOM.WW.OtOpcUa.Core.Abstractions.AlarmSeverity.High, + EventKind: "Activated", + Message: "temperature high", + User: "system", + Comment: null, + TimestampUtc: new DateTime(2026, 7, 21, 0, 0, 1, DateTimeKind.Utc)); + /// /// An enabled historian with no LocalDb registered must fail at resolution, loudly. /// From ad385eb70ff9ab53469a53682c87a8dcd6ee42af Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 06:18:57 -0400 Subject: [PATCH 10/13] docs(redundancy): record the per-Akka-cluster role-election limitation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Corrects two claims made while writing the Phase 2 gate record. The docker-dev rig is NOT misconfigured: central-1/central-2 carry admin,driver deliberately — the compose says so — because they are themselves a pair that runs the MAIN cluster's drivers. And a redundant pair is not a missing concept. The application Cluster (ClusterId) already is one: it owns drivers, devices and ClusterNode rows, and Phase 1's deployment-artifact cache is keyed by it precisely so a pair shares one entry. So the root cause is wider than the alarm drain. RedundancyStateActor elects one Primary per AKKA cluster while every real unit of redundancy is an APPLICATION cluster, and ServiceLevel, the alerts emit gate and the inbound device-write gate all inherit that mis-scoping. On a fleet of pairs in one Akka cluster, exactly one driver node cluster-wide services Primary-gated work. Recommendation revised in the gate doc: fix the election scope (its own change, its own gate, because it touches the safety-critical write gate), and explicitly do NOT derive pair identity from LocalDb:Replication:PeerAddress — that would add a second parallel notion of 'my partner' that the real fix obsoletes, and the library has no initiator flag, so both halves would have to dial just to obtain a config value. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- docs/Redundancy.md | 17 ++++++ .../2026-07-20-localdb-phase2-live-gate.md | 55 ++++++++++++++----- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/docs/Redundancy.md b/docs/Redundancy.md index 0a0dff1d..a0b5f3ad 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -69,6 +69,23 @@ Roles come from `RedundancyStateActor.BuildSnapshot`: a node with the `driver` role is `Primary` when it holds the `driver` role-leader lease, otherwise `Secondary`; a node without the `driver` role is `Detached`. +> **KNOWN LIMITATION — the election is per *Akka* cluster, not per application `Cluster`.** +> `RoleLeader("driver")` yields exactly **one** Primary across the whole Akka cluster. A fleet that +> runs several application clusters (each with its own redundant pair) inside one Akka cluster — +> which is what `docker-dev/docker-compose.yml` models, with MAIN, SITE-A and SITE-B — therefore has +> one Primary among *all* its driver nodes, not one per pair. Every consumer of the role inherits +> this: `ServiceLevel`, the `alerts` emit gate, and the inbound device-write gate below. +> +> The unit of redundancy everywhere else in the product is the application `Cluster` (`ClusterId`) — +> it owns drivers, devices and `ClusterNode` rows, and the LocalDb deployment-artifact cache is +> already keyed by it *so that a pair shares one entry*. Scoping the election the same way is the +> fix; `RedundancyStateActor` is an admin-role singleton in the same assembly as +> `AdminOperationsActor`, which already reads and groups `ClusterNodes` by cluster. +> +> Surfaced by the LocalDb Phase 2 live gate, where it stopped the alarm-history drain on every node +> in the fleet — see `docs/plans/2026-07-20-localdb-phase2-live-gate.md`. The alarm drain no longer +> depends on this (it defers only to a node that shares its queue), but the other three gates still do. + ## Data flow ``` diff --git a/docs/plans/2026-07-20-localdb-phase2-live-gate.md b/docs/plans/2026-07-20-localdb-phase2-live-gate.md index 8016bf18..8328cca2 100644 --- a/docs/plans/2026-07-20-localdb-phase2-live-gate.md +++ b/docs/plans/2026-07-20-localdb-phase2-live-gate.md @@ -130,21 +130,50 @@ the queue is pair-local. Only the dialing half of a pair knows its peer (`Replic the listening half sets `SyncListenPort` only. So the drain gate can never fully engage, and both halves of a replicated pair drain — safe, but duplicated. -Three ways out, in increasing order of scope: +### Follow-up analysis — the root cause is wider than the drain gate -- **(A) Configure both halves to dial.** Smallest change; makes the peer known on both sides so the - gate engages symmetrically. Needs confirmation that the LocalDb library accepts bidirectional dial, - and an operator-facing config change. -- **(B) Accept duplicates for unpaired-role deployments.** Change nothing further, document that the - gate only engages when redundancy roles are configured per pair. Cheapest; leaves the steady-state - double-delivery the gate was introduced to remove. -- **(C) Scope redundancy roles per pair in `RedundancyStateActor`.** Fixes the root cause for *every* - consumer of the role (ServiceLevel, scripted-alarm emit gate, inbound-write gate), all of which - inherit the same cluster-wide assumption. Largest blast radius, and the only one that makes - "Primary" mean what the docs say it means in a multi-pair fleet. +Two things assumed while writing the section above turned out to be wrong, and correcting them +changes the recommendation. -The branch is safe in every topology as it stands — no configuration loses data — so this is a -question of when to reclaim the de-duplication benefit, not an outstanding hazard. +**The rig is not misconfigured.** `central-1`/`central-2` carry `admin,driver` deliberately — the +compose says so in as many words ("central is admin,driver"), because they are themselves a +redundant pair that also runs the MAIN cluster's drivers. Stripping the driver role would change what +the rig models, not fix anything. + +**A "pair" is not a missing concept — it already exists, as the application Cluster.** OtOpcUa's +`Cluster`/`ClusterId` (the config-DB entity that owns drivers, devices and `ClusterNode` rows) is +exactly the unit a redundant pair belongs to. Phase 1 already relies on this: the deployment-artifact +cache is *keyed by ClusterId so a pair shares one entry*, and the rig's three application clusters +(MAIN, SITE-A, SITE-B) are precisely its three pairs. + +So the real defect is that **`RedundancyStateActor` elects one Primary per *Akka* cluster, while +every meaningful unit of redundancy is an *application* cluster.** The rig runs six driver nodes +across three application clusters in one Akka cluster — a topology the product otherwise supports +throughout — and exactly one of those six can ever be Primary. + +That mis-scoping is not confined to the alarm drain. Every consumer of the role inherits it: +`ServiceLevel`, the `alerts` emit gate, and the inbound device-write gate. On a multi-cluster fleet +today, only one driver node cluster-wide will service Primary-gated work at all. + +### Revised recommendation + +**(C), as a separate change with its own live gate — and explicitly *not* (A).** + +- (C) is no longer "invent a pair concept"; it is "elect per `ClusterId` instead of per Akka cluster", + using a first-class entity that already exists. It is feasible where it needs to be: + `RedundancyStateActor` has no DB access today, but it is an admin-role singleton in the same + assembly as `AdminOperationsActor`, which already reads and groups `ClusterNodes` by cluster. +- **(A) is now actively undesirable.** Deriving pair identity from `LocalDb:Replication:PeerAddress` + would introduce a *second*, parallel notion of "who is my partner" that (C) immediately obsoletes — + and it cannot be done cleanly anyway: the library exposes no initiator/passive flag, so making both + halves declare a peer means both halves *dial*, opening two duplex sessions purely to obtain a + configuration value. +- (C) touches the inbound device-write gate, which is safety-critical (the archreview 03/S4 + dual-primary fix). It therefore belongs in its own change, with its own live gate, rather than + bolted onto Phase 2. + +Until then this branch is safe in every topology — no configuration loses data — so what remains is +reclaiming the de-duplication benefit, not an outstanding hazard. ## Not merged From f862804a35175e414d73502996e4f72ccd3476cb Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 06:46:17 -0400 Subject: [PATCH 11/13] =?UTF-8?q?docs(design):=20per-cluster=20Akka=20mesh?= =?UTF-8?q?=20=E2=80=94=20separate=202-node=20mesh=20per=20application=20C?= =?UTF-8?q?luster?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design for review, mirroring the ScadaBridge hub-and-spoke shape. Researched from the ScadaBridge tree directly rather than from its CLAUDE.md, which understates the boundary in five ways: there are THREE transports (ClusterClient, gRPC, plus token-gated HTTP), the gRPC direction is inverted (central dials INTO each site), active node is the OLDEST Up member and explicitly never the cluster leader, all clusters share one ActorSystem name, and site nodes carry two roles. Records a defect worth fixing regardless of the topology decision: OtOpcUa uses two different node-selection rules at once. SBR is keep-oldest and singletons are placed on the oldest member, but RedundancyStateActor elects Primary from RoleLeader("driver") — lowest address. Those diverge permanently after a restart-and-rejoin, so the node the SBR protects and hosts every singleton on need not be the node the data-plane gates consider Primary. ScadaBridge forbids that rule by name, for the reason it gives: both sides claim leadership during a partition, which is the dual-primary shape archreview 03/S4 exists to prevent. Three open questions are left explicitly undecided: whether to adopt their autonomous-site data architecture or keep the shared ConfigDb; what to do about the registered two-node keep-oldest total-outage gap they acknowledge but have not closed; and whether to authenticate transports they left unauthenticated. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- .../2026-07-21-per-cluster-mesh-design.md | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 docs/plans/2026-07-21-per-cluster-mesh-design.md diff --git a/docs/plans/2026-07-21-per-cluster-mesh-design.md b/docs/plans/2026-07-21-per-cluster-mesh-design.md new file mode 100644 index 00000000..f19b7ebf --- /dev/null +++ b/docs/plans/2026-07-21-per-cluster-mesh-design.md @@ -0,0 +1,222 @@ +# Per-Cluster Akka Mesh — design + +> **Status:** design for review. No implementation planned yet. +> **Decision requested:** the three open questions in §6, before any per-phase plan is written. + +## 1. What this changes + +Today OtOpcUa runs **one Akka mesh containing every node of every application Cluster**. That was a +deliberate choice — `docs/plans/2026-06-07-per-cluster-scoping.md`, "Per-ClusterId Scoping +(hub-and-spoke **single mesh**)" — so the deploy channel could stay in-mesh: AdminUI → +`admin-operations` singleton → `deployments` topic → every node filters to its own `ClusterId`. + +The proposal is to move to the sister project's shape: **one Akka mesh per application Cluster, +two nodes maximum**, with central and clusters joined by explicit transports instead of cluster +gossip. + +The motivation is that redundancy becomes correct *by construction*. Every Primary-gated decision +(inbound device writes, native alarm acks, fleet alerts, ServiceLevel, and — since Phase 2 — the +alarm-history drain) currently asks "am I the driver Primary?" of a **cluster-wide** election, while +every resource being gated is **pair-local**. Splitting the mesh collapses the two scopes into one +and the question stops being ambiguous. + +## 2. What ScadaBridge actually does + +Researched directly from `/Users/dohertj2/Desktop/ScadaBridge`. Five things differ from the +summary in its own CLAUDE.md, and they matter: + +**Three transports cross the boundary, not two.** + +| Transport | Direction | Carries | +|---|---|---| +| Akka **ClusterClient** | central → site (+ replies) | command/control: deploy notify, lifecycle, queries, subscribe handshakes | +| **gRPC** server-streaming | **central dials into the site** | real-time attribute + alarm events | +| Plain **HTTP**, token-gated | site → central | the deployment config itself (notify-and-fetch) | + +**The gRPC direction is inverted from the obvious guess.** Data flows site→central, but *each site +node hosts the gRPC server* (Kestrel h2c :8083) and *central is the client*. There is no gRPC server +on central at all. + +**Active node = OLDEST Up member, explicitly not the cluster leader.** +`ActiveNodeEvaluator.SelfIsOldestUp` is documented as *"THE single definition of 'active node'"*: + +> *"Cluster LEADERSHIP (lowest address) is an Akka-internal concept that diverges from singleton +> placement permanently once the original first node restarts and rejoins; every product-level +> active/standby decision must use this evaluator, never `cluster.State.Leader`."* + +The equivalence **oldest-Up == where `ClusterSingletonManager` places singletons** *is* the design. + +**All clusters share one ActorSystem name** (`"scadabridge"`, hardcoded). They are separate clusters +only by seed-node partitioning — required, because Akka.Remote address matching means ClusterClient +could not reach a differently-named system. + +**Site nodes carry two roles:** `"Site"` and `"site-{SiteId}"`. Singletons scope to the +**site-specific** role. + +Other load-bearing details: + +- **Central discovers sites from the database** (a `Site` entity with `NodeAAddress`/`NodeBAddress` + + `GrpcNodeAAddress`/`GrpcNodeBAddress`), refreshed every 60 s and on admin change. Sites discover + central from **appsettings** — static, restart required. The asymmetry is deliberate. +- **Exactly one actor is exposed per side** via `ClusterClientReceptionist.RegisterService` — + `/user/central-communication` and `/user/site-communication` — registered **per node, not as a + singleton**, so contact rotation reaches whichever node answers. +- **No central buffering when a site is unreachable.** The send is dropped with a warning and the + caller's Ask times out. *"It keeps the central coordinator stateless with respect to site + availability."* A `ConnectionStateChanged` mechanism was built for this and **deleted as dead + code**. +- **Every unhandled message replies with a typed failure rather than dropping**, so a central Ask + can never stall on a missing handler. +- **Sender preservation is the whole Ask idiom**: `_client.Tell(new ClusterClient.Send(path, msg), + Sender)` routes the reply straight back to the waiting Ask, bypassing the comm actor. + +### The registered outage gap — read before copying + +> `Component-ClusterInfrastructure.md:113`: *"Only the FIRST seed listed in `Cluster:SeedNodes` may +> self-join to form a *new* cluster… a lone restarted non-first-seed node (with the first seed still +> down) loops on `InitJoin` forever — never `Up`, never routable. This is why the two-node +> keep-oldest **oldest/active-node crash is a total-outage gap**: after the oldest dies the younger +> survivor self-downs, and it cannot re-bootstrap alone. Recovery is operator-driven."* + +Their failover drill has a mode that exists *"to make the registered gap observable — not to pretend +it is covered."* **A two-node keep-oldest mesh has an acknowledged total-outage hole.** If OtOpcUa +adopts 2-node meshes, this must be decided deliberately, not inherited. + +### Both inter-cluster transports are unauthenticated + +Akka remoting has no TLS, no secure cookie, no `trusted-selection-paths`. The gRPC listener is h2c +with no auth covering `SiteStreamService` — including two Pull RPCs that return audit rows. The only +authenticated pieces are the HTTP fetch token and `LocalDbSyncAuthInterceptor`, which gates *only* +`/localdb_sync.v1.LocalDbSync/`. Their boundary assumes a trusted network. + +**OtOpcUa already ships that same fail-closed interceptor** (LocalDb Phase 1) — it is a ready-made +template if we choose to authenticate more than they did. + +## 3. What OtOpcUa has today + +**Nine cross-node DPS topics plus a singleton**, all currently relying on one mesh: + +| Channel | Direction | Purpose | +|---|---|---| +| `deployments` / `deployment-acks` | central ↔ nodes | deploy dispatch + acks | +| `driver-control` | central → nodes | AdminUI Reconnect/Restart | +| `redundancy-state` | central → nodes | roles + peer probe results | +| `alerts` | nodes → central | fleet alarms → `/alerts` | +| `driver-health`, `driver-resilience-status`, `fleet-status`, `script-logs` | nodes → central | AdminUI live panels | +| `admin-operations` singleton | central | operator ack/shelve into engines | + +Three facts make the split cheaper than that table suggests: + +1. **The deploy path is already notify-and-fetch.** `DispatchDeployment` carries only + `DeploymentId` + `RevisionHash` + `CorrelationId`; the node fetches the artifact itself. We + independently arrived at the pattern ScadaBridge had to retrofit, and we do **not** carry its + 128 KB Akka frame exposure on this path. +2. **Deploy state already has a DB substrate.** `ConfigPublishCoordinator` persists per-node ACKs to + `NodeDeploymentState` so a singleton failover recovers in-flight state from the DB. +3. **`ClusterNode` rows already enumerate every node per application Cluster** — the same table + `AdminOperationsActor` groups by cluster. This replaces the coordinator's one genuinely + mesh-bound dependency: it derives its expected-ack set from `Akka.Cluster.State.Members` filtered + by role. + +So the deploy channel needs only a small notify + ack transport. **The nine live-telemetry channels +are the real work**, and they are exactly what ScadaBridge built ClusterClient + gRPC for. + +## 4. A defect to fix regardless of this design + +OtOpcUa currently uses **two different node-selection rules at once**: + +- Split-brain resolution is **keep-oldest** (`Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf:49`). +- `ClusterSingletonManager` places singletons on the **oldest** member. +- But `RedundancyStateActor.BuildSnapshot` elects Primary from **`RoleLeader("driver")`** — lowest + address among driver members. + +Those select the same node only by coincidence. After a restart-and-rejoin the role leader and the +oldest member diverge permanently, which means the node the SBR protects and hosts every singleton +on need not be the node the data-plane gates consider Primary. This is precisely the rule ScadaBridge +forbids by name, and for the reason it gives: *"both sides claim leadership during a partition"* — +the dual-primary shape archreview 03/S4 exists to prevent. + +**Switching the role derivation to oldest-Up-with-role is small, is correct under either topology, +and should not wait for this design.** It also happens to be exactly what the separate-mesh model +needs, so it is not throwaway work. + +## 5. Target architecture + +Mirror ScadaBridge's shape, adapted to OtOpcUa's existing substrate: + +- **One Akka mesh per application `Cluster`, two nodes max.** Same ActorSystem name (`otopcua`) + everywhere; separation by seed-node partitioning only. +- **Roles per node:** `driver` plus a cluster-specific `cluster-{ClusterId}`, with singletons scoped + to the cluster-specific role. +- **Active node = oldest Up member with role** (§4), one definition, used by the data-plane gates, + ServiceLevel, and the alarm drain alike. +- **Central ↔ cluster command/control over ClusterClient**, with exactly one receptionist-registered + actor per side and central discovering cluster node addresses from `ClusterNode` rows (extended + with Akka + gRPC addresses, mirroring their `Site` entity). +- **Deploy stays notify-and-fetch.** The notify crosses via ClusterClient; the artifact continues to + come from the shared ConfigDb (see §6.1). The coordinator's expected-ack set comes from + `ClusterNode` rows instead of cluster membership. +- **Live telemetry over gRPC**, central dialling into each cluster node, replacing the seven + observability topics with one stream contract carrying a `oneof` event — additive-only field + evolution, contract-locked by test. + +Deliberately **not** copied: their transient-only central alarm cache and site-local-primary storage +(see §6.1), and their unauthenticated transports (§6.3). + +## 6. Open questions — these need answering before a plan + +### 6.1 Shared ConfigDb, or autonomous clusters? + +"Mimic ScadaBridge" splits into two very different commitments: + +- **Topology** — separate mesh per cluster. Contained, given §3. +- **Data architecture** — ScadaBridge sites are *autonomous*: their own SQLite, config fetched from + central over token-gated HTTP, because a site is not assumed to reach central's database at all. + +OtOpcUa's premise is the opposite: one shared ConfigDb every node reads directly, with LocalDb as an +**outage cache** — the entire basis of Phase 1's boot-from-cache. + +**Recommendation: take the topology, keep the shared ConfigDb.** Sites are autonomous in ScadaBridge +because they genuinely cannot rely on reaching central. If OtOpcUa's nodes can — and Phase 1 assumes +they normally can — then site-local-primary buys a large amount of machinery (staged pending rows, +fetch tokens, TTL purge singleton, reconciliation service) for a problem we do not have. Adopting it +would also reverse the premise of the LocalDb work just shipped. + +### 6.2 The two-node keep-oldest outage gap + +Inheriting 2-node keep-oldest inherits the registered hole in §2: the younger survivor cannot +re-bootstrap alone after the oldest dies. Options: accept it with an operator runbook (their choice); +add a third lightweight seed/witness per cluster; or use a different downing strategy. **This needs +an explicit decision — it is a total-outage hole, not a degradation.** + +### 6.3 Authentication on the new transports + +Their boundary assumes a trusted network. OtOpcUa's clusters may span less trusted links. If so we +need what they did not build: Akka.Remote TLS and a gRPC interceptor covering the whole service +surface. `LocalDbSyncAuthInterceptor` is already in our tree as a fail-closed template. + +## 7. Sequencing sketch + +Deliberately not a task plan — per-phase plans follow once §6 is settled. + +| Phase | Content | Independent of the split? | +|---|---|---| +| 0 | Oldest-Up role derivation (§4) | **Yes** — ship first | +| 1 | `ClusterNode` gains Akka + gRPC address columns; coordinator sources its expected-ack set from the DB | Yes | +| 2 | Comm actors + receptionist registration; ClusterClient transport; deploy notify + acks across the boundary | No | +| 3 | gRPC stream contract; migrate the seven observability topics | No | +| 4 | Mesh partition: per-cluster seed nodes, cluster-scoped roles + singletons; rig models the real topology | No | +| 5 | Auth per §6.3; failover drill; live gate | No | + +## 8. Risks + +- **The rig currently models the topology that would be abolished.** Phase 4 rewrites + `docker-dev/docker-compose.yml` substantially, and every live gate that depends on it. +- **Losing gossip loses free fleet-wide observability.** Seven AdminUI panels are fed by DPS today; + each needs an explicit stream and a reconnect story. +- **Akka frame size.** We are clean on the deploy path, but anything new that carries payload over + ClusterClient inherits the 128 KB default with `log-frame-size-exceeding` off — a silent + single-message drop that leaves the association healthy. If we send payload at all, set both. +- **Do not stack application-level last-write-wins on LocalDb's HLC.** Their `SiteStorageService` + carries this warning in code. Verified: our `deployment_pointer` upsert is unconditional and the + alarm sink is idempotent on a payload hash, so we are currently clean — keep it that way. From fe12ed9d3467edb8159192c468d22f17afcd8245 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 10:07:00 -0400 Subject: [PATCH 12/13] docs(design): settle the three open questions on the per-cluster mesh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (1) One central DB, driver nodes disconnected from it. Verified in ScadaBridge before adopting: AddConfigurationDatabase is called at Program.cs:264, inside the Central branch (89-478); the Site branch starts at 479, so site nodes never register the central database at all. There is still exactly one authoritative configuration database — it is central-only, and sites fetch from central and cache locally. The consequence worth naming: this promotes LocalDb from an outage cache to the driver node's steady-state configuration store. Boot-from-cache stops being the exceptional path and becomes the normal one. Phase 1's chunking, SHA-256 verification, retention and pair replication all carry over — but a cache defect that used to be a degraded-mode bug becomes a total-availability one, so the #485 unreadable-artifact class now has a larger blast radius. Audited the five driver-side ConfigDb consumers that must be re-homed. Two are more than mechanical: EfAlarmConditionStateStore holds pair-local Part 9 condition state and should follow the alarm S&F buffer into LocalDb, and DbHealthProbeActor feeds ServiceLevel — a driver node with no DB to probe changes what a client-visible value means. (2) The keep-oldest gap: corrected the status rather than the conclusion. What was resolved is the documentation and the drill, not the hole. Gitea PR #12 (closed) carried T1 badc97af, which made the drill "measure the registered keep-oldest outage instead of pretending recovery", and T2 d5364506, which dropped "the undeliverable ~25s active-crash promise". The gap itself is still the registered deferred keep-oldest topology decision on the 2026-07-08 master tracker, with no open issue. Accepted here with the same posture, recorded as a known risk. (3) Auth matches ScadaBridge for now — unauthenticated inter-cluster transports on a trusted network, recorded as an accepted risk with the fail-closed interceptor already in our tree noted as the template for whenever it is revisited. Sequencing grows from six phases to eight; the two that change a running system's data path rather than its wiring each get their own live gate. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- .../2026-07-21-per-cluster-mesh-design.md | 106 +++++++++++++----- 1 file changed, 77 insertions(+), 29 deletions(-) diff --git a/docs/plans/2026-07-21-per-cluster-mesh-design.md b/docs/plans/2026-07-21-per-cluster-mesh-design.md index f19b7ebf..89f1e22e 100644 --- a/docs/plans/2026-07-21-per-cluster-mesh-design.md +++ b/docs/plans/2026-07-21-per-cluster-mesh-design.md @@ -1,7 +1,9 @@ # Per-Cluster Akka Mesh — design > **Status:** design for review. No implementation planned yet. -> **Decision requested:** the three open questions in §6, before any per-phase plan is written. +> **Decisions settled 2026-07-21** — see §6: one central DB with driver nodes disconnected from it, +> the two-node keep-oldest gap accepted as ScadaBridge accepts it, and their auth posture matched for +> now. ## 1. What this changes @@ -153,9 +155,12 @@ Mirror ScadaBridge's shape, adapted to OtOpcUa's existing substrate: - **Central ↔ cluster command/control over ClusterClient**, with exactly one receptionist-registered actor per side and central discovering cluster node addresses from `ClusterNode` rows (extended with Akka + gRPC addresses, mirroring their `Site` entity). -- **Deploy stays notify-and-fetch.** The notify crosses via ClusterClient; the artifact continues to - come from the shared ConfigDb (see §6.1). The coordinator's expected-ack set comes from +- **Deploy stays notify-and-fetch, but the fetch changes source.** The notify crosses via + ClusterClient; the artifact then comes **from central**, not from the ConfigDb the driver node no + longer connects to (§6.1), and is cached in LocalDb. The coordinator's expected-ack set comes from `ClusterNode` rows instead of cluster membership. +- **Driver nodes hold no ConfigDb connection.** LocalDb becomes their steady-state configuration + store; five current DB consumers are re-homed or re-sourced per §6.1. - **Live telemetry over gRPC**, central dialling into each cluster node, replacing the seven observability topics with one stream contract carrying a `oneof` event — additive-only field evolution, contract-locked by test. @@ -163,53 +168,96 @@ Mirror ScadaBridge's shape, adapted to OtOpcUa's existing substrate: Deliberately **not** copied: their transient-only central alarm cache and site-local-primary storage (see §6.1), and their unauthenticated transports (§6.3). -## 6. Open questions — these need answering before a plan +## 6. Decisions (settled 2026-07-21) -### 6.1 Shared ConfigDb, or autonomous clusters? +### 6.1 DECIDED — one central DB, and driver nodes never connect to it -"Mimic ScadaBridge" splits into two very different commitments: +**Verified in ScadaBridge before adopting.** `Program.cs` calls +`AddConfigurationDatabase(configDbConnectionString)` at line 264 — inside the Central branch +(89–478). The Site branch begins at line 479. **Site nodes never register the central configuration +database at all**; they receive config from central (notify → token-gated HTTP fetch) and persist it +locally. There is still exactly one authoritative configuration database — it is simply +central-only. -- **Topology** — separate mesh per cluster. Contained, given §3. -- **Data architecture** — ScadaBridge sites are *autonomous*: their own SQLite, config fetched from - central over token-gated HTTP, because a site is not assumed to reach central's database at all. +OtOpcUa adopts the same shape: **the ConfigDb stays the single source of truth, and driver nodes stop +connecting to it.** They obtain their configuration from the central nodes and cache it locally in +LocalDb. -OtOpcUa's premise is the opposite: one shared ConfigDb every node reads directly, with LocalDb as an -**outage cache** — the entire basis of Phase 1's boot-from-cache. +**This promotes LocalDb from an outage cache to the driver node's steady-state configuration store.** +That is a reframing of the Phase 1 work rather than a contradiction of it: boot-from-cache stops +being the exceptional path and becomes the normal one, and "central SQL unreachable" stops being a +degraded mode for driver nodes because they never held a connection to lose. The chunking, +SHA-256 verification, newest-2 retention and pair replication all carry over unchanged. -**Recommendation: take the topology, keep the shared ConfigDb.** Sites are autonomous in ScadaBridge -because they genuinely cannot rely on reaching central. If OtOpcUa's nodes can — and Phase 1 assumes -they normally can — then site-local-primary buys a large amount of machinery (staged pending rows, -fetch tokens, TTL purge singleton, reconciliation service) for a problem we do not have. Adopting it -would also reverse the premise of the LocalDb work just shipped. +**Scope — five driver-side ConfigDb consumers must be re-homed or re-sourced.** Audited: -### 6.2 The two-node keep-oldest outage gap +| Consumer | Current use | Disposition | +|---|---|---| +| `DriverHostActor` | artifact fetch + ack writes (5 `CreateDbContext` sites) | artifact via fetch-and-cache; acks via the ack transport | +| `EfAlarmConditionStateStore` | Part 9 alarm condition state | **move to LocalDb** — it is pair-local state, the same journey Phase 2 made for the alarm S&F buffer | +| `DbHealthProbeActor` | DB reachability feeding ServiceLevel tiering | **semantics change** — a driver node has no DB to probe; decide what health input replaces it | +| `OpcUaPublishActor` | (audit required) | TBD | +| `ServiceCollectionExtensions` | registration | follows the above | -Inheriting 2-node keep-oldest inherits the registered hole in §2: the younger survivor cannot -re-bootstrap alone after the oldest dies. Options: accept it with an operator runbook (their choice); -add a third lightweight seed/witness per cluster; or use a different downing strategy. **This needs -an explicit decision — it is a total-outage hole, not a degradation.** +`DbHealthProbeActor` deserves particular care: DB health is currently a ServiceLevel input, and +under this model it stops existing for driver nodes. That is a behaviour change on a client-visible +value, not just an internal refactor. -### 6.3 Authentication on the new transports +### 6.2 DECIDED — accept the two-node keep-oldest gap, as ScadaBridge does -Their boundary assumes a trusted network. OtOpcUa's clusters may span less trusted links. If so we -need what they did not build: Akka.Remote TLS and a gRPC interceptor covering the whole service -surface. `LocalDbSyncAuthInterceptor` is already in our tree as a fail-closed template. +**Status corrected.** What was resolved is the *documentation and the drill*, not the gap. Gitea +**PR #12** (closed) — — "R2-01: Cluster, +Host & Failover round-2 fixes" carried: + +- **T1** (`badc97af`) — the failover drill now kills the STANDBY by default, and its `active` mode + *"measures the registered keep-oldest outage instead of pretending recovery"*. +- **T2** (`d5364506`) — per-direction failover docs, dropping what the commit message calls *"the + undeliverable ~25s active-crash promise"*. + +So the closed work made the gap **honest**, not absent. `Component-ClusterInfrastructure.md:113` +still reads: *"Removing the gap itself is the **registered deferred keep-oldest topology/strategy +decision** (master tracker 2026-07-08, owner: user)."* There is no open Gitea issue for it — it is +tracked on that master tracker, not in the issue list. + +OtOpcUa inherits the same gap and, for now, the same posture. Worth recording so it is a known +accepted risk rather than a surprise: after the oldest node of a two-node mesh dies, the survivor +self-downs and cannot re-bootstrap alone, because only the first-listed seed may self-join. Recovery +is operator-driven — restart the dead first seed, or restart the survivor with a self-first seed +override. + +### 6.3 DECIDED — match ScadaBridge's auth posture for now + +Inter-cluster transports stay unauthenticated and unencrypted, as theirs are: the boundary assumes a +trusted network. Recorded as an accepted risk rather than an oversight, with two notes for whenever +it is revisited: `LocalDbSyncAuthInterceptor` is already in our tree as a fail-closed template, and +the surface most worth gating first is any Pull-style RPC that returns historical rows. ## 7. Sequencing sketch -Deliberately not a task plan — per-phase plans follow once §6 is settled. +Deliberately not a task plan — per-phase plans follow, one at a time. | Phase | Content | Independent of the split? | |---|---|---| | 0 | Oldest-Up role derivation (§4) | **Yes** — ship first | | 1 | `ClusterNode` gains Akka + gRPC address columns; coordinator sources its expected-ack set from the DB | Yes | | 2 | Comm actors + receptionist registration; ClusterClient transport; deploy notify + acks across the boundary | No | -| 3 | gRPC stream contract; migrate the seven observability topics | No | -| 4 | Mesh partition: per-cluster seed nodes, cluster-scoped roles + singletons; rig models the real topology | No | -| 5 | Auth per §6.3; failover drill; live gate | No | +| 3 | Config fetch-and-cache: artifact served by central, driver nodes read LocalDb (§6.1) | No | +| 4 | Cut the driver-side ConfigDb connection: re-home `EfAlarmConditionStateStore`, resolve the `DbHealthProbeActor` ServiceLevel input, audit `OpcUaPublishActor` | No | +| 5 | gRPC stream contract; migrate the seven observability topics | No | +| 6 | Mesh partition: per-cluster seed nodes, cluster-scoped roles + singletons; rig models the real topology | No | +| 7 | Failover drill (both directions, per §6.2); live gate | No | + +Phases 3 and 4 are the ones that change a running system's data path rather than its wiring, and each +deserves its own live gate. ## 8. Risks +- **LocalDb becomes load-bearing for configuration, not just resilient.** Phase 1 built it as a + fallback; §6.1 makes it the driver node's only config store. A cache defect that was previously a + degraded-mode bug becomes a total-availability bug — the #485 unreadable-artifact class, with the + blast radius raised. +- **`DbHealthProbeActor` feeds a client-visible value.** Removing the driver-side DB connection + changes what ServiceLevel means on those nodes; that is an interop-visible change, not internal. - **The rig currently models the topology that would be abolished.** Phase 4 rewrites `docker-dev/docker-compose.yml` substantially, and every live gate that depends on it. - **Losing gossip loses free fleet-wide observability.** Seven AdminUI panels are fed by DPS today; From 4caaa11e9ec85df339fb38f831cad64736cebdfe Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 10:09:10 -0400 Subject: [PATCH 13/13] docs(design): fix two staleness artefacts the settled decisions introduced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Section 5 still listed "site-local-primary storage" among the things deliberately NOT copied from ScadaBridge — which §6.1 now reverses, since driver nodes stop connecting to the ConfigDb and read their configuration from LocalDb. Replaced with an explicit copied/not-copied split, and narrowed the not-copied item to what it should have said all along: their transient-only central alarm cache, which OtOpcUa has no reason to adopt because it persists alarm history through the historian and Phase 2's replicated buffer already survives a failover. Also corrected a renumbering artefact — the risk about rewriting the docker-dev rig cited Phase 4, but the mesh partition moved to Phase 6 when the two data-path phases were inserted. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- docs/plans/2026-07-21-per-cluster-mesh-design.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/plans/2026-07-21-per-cluster-mesh-design.md b/docs/plans/2026-07-21-per-cluster-mesh-design.md index 89f1e22e..addeb661 100644 --- a/docs/plans/2026-07-21-per-cluster-mesh-design.md +++ b/docs/plans/2026-07-21-per-cluster-mesh-design.md @@ -165,8 +165,14 @@ Mirror ScadaBridge's shape, adapted to OtOpcUa's existing substrate: observability topics with one stream contract carrying a `oneof` event — additive-only field evolution, contract-locked by test. -Deliberately **not** copied: their transient-only central alarm cache and site-local-primary storage -(see §6.1), and their unauthenticated transports (§6.3). +**Copied:** their node-local configuration store fed by fetch-and-cache (§6.1), their oldest-Up +active-node rule (§4), their two-node keep-oldest posture including its acknowledged outage gap +(§6.2), and their unauthenticated inter-cluster transports (§6.3). + +**Deliberately not copied:** their *transient-only* central alarm cache. OtOpcUa persists alarm +history centrally through the historian, and Phase 2's replicated store-and-forward buffer already +guarantees delivery across a failover — so there is no reason to adopt a design whose stated +trade-off is that a new active node re-seeds alarm state from scratch. ## 6. Decisions (settled 2026-07-21) @@ -258,7 +264,7 @@ deserves its own live gate. blast radius raised. - **`DbHealthProbeActor` feeds a client-visible value.** Removing the driver-side DB connection changes what ServiceLevel means on those nodes; that is an interop-visible change, not internal. -- **The rig currently models the topology that would be abolished.** Phase 4 rewrites +- **The rig currently models the topology that would be abolished.** Phase 6 rewrites `docker-dev/docker-compose.yml` substantially, and every live gate that depends on it. - **Losing gossip loses free fleet-wide observability.** Seven AdminUI panels are fed by DPS today; each needs an explicit stream and a reconnect story.