Compare commits
10 Commits
0b5e9b44f6
...
b64857c8d6
| Author | SHA1 | Date | |
|---|---|---|---|
| b64857c8d6 | |||
| 568735a41e | |||
| 95c108f1d7 | |||
| 9cf7c0f007 | |||
| 3c395d794a | |||
| 98b94771ae | |||
| 59c695191c | |||
| e62b076f2e | |||
| a560e9eaf9 | |||
| 0d8a80aa27 |
@@ -120,6 +120,12 @@ Other peers in the `scadaproj` family (see `scadaproj/CLAUDE.md` for details): `
|
||||
- DCL write failures returned synchronously to calling script.
|
||||
- Tag path resolution retried periodically for devices still booting.
|
||||
- Static attribute writes persisted to local SQLite (survive restart/failover, reset on redeployment).
|
||||
- **Consolidated site database (LocalDb Phase 1, 2026-07-19).** `OperationTracking` and `site_events` now live in ONE `ZB.MOM.WW.LocalDb`-managed SQLite file, configured by the **required** `LocalDb:Path` (`/app/data/site-localdb.db` on the rig; validated with `ValidateOnStart`, so a site config missing it fails to boot). Both are `RegisterReplicated` tables. Consequences worth knowing:
|
||||
- `site_events.id` changed from autoincrement INTEGER to an application-minted **GUID**. Last-writer-wins keys on the primary key, so two nodes independently minting `id=1,2,3…` would destroy each other's events rather than merge them. The event-log read path uses a composite `(timestamp, id)` keyset cursor with an **opaque string** continuation token; `EventLogEntry.Id` and both `ContinuationToken`s are `string`/`string?` on the site↔central Akka DTOs.
|
||||
- `ScadaBridge:OperationTracking:ConnectionString` and `ScadaBridge:SiteEventLog:DatabasePath` are **migration-only** — nothing reads them but `SiteLocalDbLegacyMigrator`, which copies a pre-Phase-1 file in once (deterministic `mig-{NodeName}-{legacyId}` event ids, `INSERT OR IGNORE`, runs AFTER `RegisterReplicated` so migrated rows replicate) and renames it `.migrated`. Delete the keys once a node has migrated.
|
||||
- This incidentally fixes a data-loss bug: both legacy databases defaulted to CWD-relative paths **outside** the mounted volume and were discarded on every container recreate.
|
||||
- **Replication is default-OFF and opt-in** via `LocalDb:Replication:PeerAddress` + a matching `ApiKey` on both nodes. `LocalDbSyncAuthInterceptor` is **fail-closed**: no configured key means no sync stream is accepted at all, so a key typo does not degrade to unauthenticated replication — the pair simply stops converging. The sync endpoint shares the existing site gRPC h2c listener (8083); no new port. Rig posture: **site-a replicated, site-b/site-c deliberately not**, so both states are proven side-by-side. Status surfaces on the site health report as `LocalDbReplicationConnected` / `LocalDbOplogBacklog` (both nullable — null means "no data", NOT "disconnected with an empty backlog") and as `localdb_*` Prometheus series. Note `ZbTelemetryOptions.Meters` is an **allowlist** (`SiteServiceRegistration.ObservedMeters`); an unlisted meter exports nothing, silently.
|
||||
- Design: **scadaproj** `docs/plans/2026-07-19-scadabridge-localdb-design.md` (that doc lives in the umbrella repo, not here); Phase 1 plan + Phase 2 gate are here under `docs/plans/`. Phase 2 (config tables + `sf_messages`, deleting `SiteReplicationActor` + StoreAndForward `ReplicationService`) is NOT started — see `docs/plans/2026-07-19-localdb-phase2-gate.md`.
|
||||
- All timestamps are UTC throughout the system.
|
||||
- Inter-cluster communication uses two transports: ClusterClient for command/control (deployments, lifecycle, subscribe/unsubscribe handshake, snapshots) and gRPC server-streaming for real-time data (attribute values, alarm states). Both CentralCommunicationActor and SiteCommunicationActor registered with receptionist. Central creates one ClusterClient per site using NodeA/NodeB as contact points. Sites configure multiple central contact points for failover. Addresses cached in CentralCommunicationActor, refreshed periodically (60s) and on admin changes. Heartbeats serve health monitoring only.
|
||||
- gRPC streaming channel: SiteStreamGrpcServer on each site node (Kestrel HTTP/2, port 8083); central creates per-site SiteStreamGrpcClient via SiteStreamGrpcClientFactory. Site entity has GrpcNodeAAddress/GrpcNodeBAddress fields. Proto: sitestream.proto with SiteStreamService, SiteStreamEvent (oneof: AttributeValueUpdate, AlarmStateUpdate). DebugStreamEvent message removed (no longer flows through ClusterClient).
|
||||
|
||||
@@ -64,6 +64,23 @@
|
||||
// outside the volume and were lost on every recreate.
|
||||
// Replication is opt-in and configured separately; absent = local-only.
|
||||
"LocalDb": {
|
||||
"Path": "/app/data/site-localdb.db"
|
||||
"Path": "/app/data/site-localdb.db",
|
||||
// Site-a is the rig's REPLICATED pair; site-b and site-c deliberately stay
|
||||
// unreplicated so the default-OFF posture is proven side-by-side on one rig.
|
||||
//
|
||||
// This node is the initiator: it dials the peer. Replication is still
|
||||
// bidirectional - the passive node's writes flow back over the same stream -
|
||||
// so only one side needs PeerAddress. Port 8083 is the existing site gRPC
|
||||
// listener (h2c); the sync endpoint shares it, no new port.
|
||||
//
|
||||
// The ApiKey must be IDENTICAL on both nodes: the host's
|
||||
// LocalDbSyncAuthInterceptor is fail-closed, so a mismatch (or a missing key)
|
||||
// rejects every sync stream. A plain dev key here matches the rig's existing
|
||||
// posture; PRODUCTION uses a "${secret:...}" reference resolved by the
|
||||
// pre-host secret expander.
|
||||
"Replication": {
|
||||
"PeerAddress": "http://scadabridge-site-a-b:8083",
|
||||
"ApiKey": "dev-site-a-localdb-sync-key"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,6 +64,16 @@
|
||||
// outside the volume and were lost on every recreate.
|
||||
// Replication is opt-in and configured separately; absent = local-only.
|
||||
"LocalDb": {
|
||||
"Path": "/app/data/site-localdb.db"
|
||||
"Path": "/app/data/site-localdb.db",
|
||||
// The PASSIVE half of site-a's replicated pair: no PeerAddress, so this node's
|
||||
// sync initiator starts and idles while node-a dials in. Its own writes still
|
||||
// reach node-a - the stream is bidirectional.
|
||||
//
|
||||
// The key MUST match node-a's exactly. LocalDbSyncAuthInterceptor is
|
||||
// fail-closed, so a typo here does not degrade to unauthenticated replication;
|
||||
// it rejects every stream and the pair silently stops converging.
|
||||
"Replication": {
|
||||
"ApiKey": "dev-site-a-localdb-sync-key"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,21 +30,166 @@
|
||||
}
|
||||
],
|
||||
"tasks": [
|
||||
{"id": 1, "subject": "Task 1: Add LocalDb packages to the build", "status": "completed", "commit": "7370b331"},
|
||||
{"id": 2, "subject": "Task 2: Shared schema helpers (tracking + events, GUID PK)", "status": "completed", "blockedBy": [1], "commit": "727fa48c"},
|
||||
{"id": 3, "subject": "Task 3: Register LocalDb in the site composition root", "status": "completed", "blockedBy": [2], "commit": "2977e6c4", "note": "LocalDb:Path added to 10 configs (8 site nodes incl. docker-env2 site-x, dev template, wonder prod sample) - it is REQUIRED via ValidateOnStart, so any site config missing it fails startup."},
|
||||
{"id": 4, "subject": "Task 4: Rewire OperationTrackingStore onto ILocalDb connections", "status": "pending", "blockedBy": [3]},
|
||||
{"id": 5, "subject": "Task 5a: Rewire SiteEventLogger onto ILocalDb + mint GUID ids", "status": "partial", "blockedBy": [3], "commit": "727fa48c", "note": "GUID minting DONE (landed with Task 2 to avoid committing a broken writer). REMAINING: swap the owned SqliteConnection to ILocalDb.CreateConnection() - still blocked on Task 3."},
|
||||
{"id": 15, "subject": "Task 5b: Migrate the event-log read path off integer ids", "status": "completed", "blockedBy": [5], "commit": "727fa48c"},
|
||||
{"id": 6, "subject": "Task 6: One-time legacy data migrator", "status": "pending", "blockedBy": [4, 5]},
|
||||
{"id": 7, "subject": "Task 7: Replication registration + sync endpoint (default OFF)", "status": "pending", "blockedBy": [3]},
|
||||
{"id": 8, "subject": "Task 8: Bearer auth interceptor for the sync endpoint", "status": "pending", "blockedBy": [7]},
|
||||
{"id": 9, "subject": "Task 9: Surface ISyncStatus as site health signal", "status": "pending", "blockedBy": [7]},
|
||||
{"id": 10, "subject": "Task 10: Two-node in-process convergence test", "status": "pending", "blockedBy": [6, 8, 15]},
|
||||
{"id": 11, "subject": "Task 11: Docker rig enablement (site-a pair)", "status": "pending", "blockedBy": [8]},
|
||||
{"id": 12, "subject": "Task 12: Live gate on the docker rig", "status": "pending", "blockedBy": [9, 10, 11]},
|
||||
{"id": 13, "subject": "Task 13: Documentation truth pass", "status": "pending", "blockedBy": [12]},
|
||||
{"id": 14, "subject": "Task 14: Phase 2 gate - plan the bespoke-replicator migration", "status": "pending", "blockedBy": [12]}
|
||||
{
|
||||
"id": 1,
|
||||
"subject": "Task 1: Add LocalDb packages to the build",
|
||||
"status": "completed",
|
||||
"commit": "7370b331"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"subject": "Task 2: Shared schema helpers (tracking + events, GUID PK)",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
1
|
||||
],
|
||||
"commit": "727fa48c"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"subject": "Task 3: Register LocalDb in the site composition root",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
2
|
||||
],
|
||||
"commit": "2977e6c4",
|
||||
"note": "LocalDb:Path added to 10 configs (8 site nodes incl. docker-env2 site-x, dev template, wonder prod sample) - it is REQUIRED via ValidateOnStart, so any site config missing it fails startup."
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"subject": "Task 4: Rewire OperationTrackingStore onto ILocalDb connections",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
3
|
||||
],
|
||||
"commit": "0b5e9b44",
|
||||
"note": "Ctor takes ILocalDb, not IOptions<OperationTrackingOptions>; both ad-hoc reader paths switched too and their OpenAsync calls removed (CreateConnection returns an OPEN connection). Three DI fixtures needed LocalDb:Path added because resolving IOperationTrackingStore now forces ILocalDb construction."
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"subject": "Task 5a: Rewire SiteEventLogger onto ILocalDb + mint GUID ids",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
3
|
||||
],
|
||||
"commit": "727fa48c + 0d8a80aa",
|
||||
"note": "GUID minting landed early with Task 2 (727fa48c) to avoid committing a writer that violated site_events.id NOT NULL. The ILocalDb.CreateConnection() swap is 0d8a80aa. connectionStringOverride DELETED, not preserved: site_events is replicated, so a raw connection lacks zb_hlc_next() and fails closed."
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"subject": "Task 5b: Migrate the event-log read path off integer ids",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
5
|
||||
],
|
||||
"commit": "727fa48c"
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"subject": "Task 6: One-time legacy data migrator",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
4,
|
||||
5
|
||||
],
|
||||
"commit": "98b94771",
|
||||
"note": "Runs AFTER RegisterReplicated (pinned by an oplog assertion). Deterministic mig-{NodeName}-{legacyId} event ids, not fresh GUIDs, so a crash between commit and rename cannot duplicate. DEVIATION: sync transaction on a CreateConnection() connection instead of BeginTransactionAsync - OnReady is a synchronous Action<ILocalDb>."
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"subject": "Task 7: Replication registration + sync endpoint (default OFF)",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
3
|
||||
],
|
||||
"commit": "a560e9ea",
|
||||
"note": "DEVIATION: AddZbLocalDbReplication moved from Program.cs (where the plan put it) into SiteServiceRegistration, because the composition-root tests build that graph and not Program.cs - in Program.cs a missing registration would still show green. MapZbLocalDbSync stays in Program.cs (needs the WebApplication). Endpoint is UNAUTHENTICATED until Task 8; no config in this repo sets a peer."
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"subject": "Task 8: Bearer auth interceptor for the sync endpoint",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
7
|
||||
],
|
||||
"commit": "59c69519",
|
||||
"note": "Fail-closed: no configured ApiKey rejects ALL sync streams. All four handler shapes gated, not just unary (the real RPC is a duplex stream). DEVIATION: Grpc.Core.Testing does not exist on grpc-dotnet; hand-rolled FakeServerCallContext instead of adding the retired native package."
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"subject": "Task 9: Surface ISyncStatus as site health signal",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
7
|
||||
],
|
||||
"commit": "59c69519",
|
||||
"note": "Additive init properties on SiteHealthReport so the Akka DTO ctor is untouched. Both fields nullable: null = no data, false/0 = a real reading (which is the healthy default-OFF state). Collector stores one tuple, read once, so a torn read cannot pair fresh Connected with stale backlog."
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"subject": "Task 10: Two-node in-process convergence test",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
6,
|
||||
8,
|
||||
15
|
||||
],
|
||||
"commit": "3c395d79",
|
||||
"note": "5 scenarios over real loopback gRPC + the real interceptor, initialized via SiteLocalDbSetup.OnReady. Verified NON-vacuous: mismatching the two ApiKeys turns all 5 red with 2m30s of timeouts."
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"subject": "Task 11: Docker rig enablement (site-a pair)",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
8
|
||||
],
|
||||
"commit": "(with Task 11 commit)",
|
||||
"note": "site-a replicated, site-b/site-c deliberately not, so default-OFF is proven side-by-side on one rig."
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"subject": "Task 12: Live gate on the docker rig",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
9,
|
||||
10,
|
||||
11
|
||||
],
|
||||
"commit": "95c108f1",
|
||||
"note": "PASS. Found one REAL defect: ZbTelemetryOptions.Meters is a silent allowlist, so localdb_* was absent from /metrics - the plan's 'flows through automatically' claim was wrong. Fixed + pinned. Evidence: identical row_version/HLC/originating node_id across the pair, 0 dead letters, oplog drained, site failover + rejoin catch-up, localdb_* scraped, site-b inert. NOTE: docker/failover-drill.sh targets CENTRAL nodes and does NOT exercise the site-pair claim; the site flip was run directly."
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"subject": "Task 13: Documentation truth pass",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
12
|
||||
],
|
||||
"commit": "(docs commit)",
|
||||
"note": "Both CLAUDE.md files + the false 'starts a fresh log' XML doc + the two now-migration-only option properties."
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"subject": "Task 14: Phase 2 gate - plan the bespoke-replicator migration",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
12
|
||||
],
|
||||
"commit": "(gate commit)",
|
||||
"note": "Gate document only, per the plan - Phase 2 gets its own implementation plan. docs/plans/2026-07-19-localdb-phase2-gate.md. KEY FINDING: the rig SOAK that Phase 2's oplog-cap tuning needs has NOT been run; the live gate proved correctness, not steady-state behaviour."
|
||||
}
|
||||
],
|
||||
"lastUpdated": "2026-07-19"
|
||||
}
|
||||
"knownFlakes": [
|
||||
{
|
||||
"test": "SiteRuntime.Tests InstanceActorChildAttributeRaceTests.ChildActors_AreSeededFromAnIsolatedCopy_NotTheLiveAttributesDictionary",
|
||||
"note": "Intermittent ActorNotFoundException under full-suite load; passes in isolation and in most full runs. Pre-existing, unrelated to LocalDb. Worth its own issue."
|
||||
},
|
||||
{
|
||||
"test": "AuditLog.Tests ParentExecutionIdCorrelationTests.InboundRoutedRun_AllRoutedRows_CarryInboundExecutionId_AsParentExecutionId",
|
||||
"note": "Cold-MSSQL-fixture timing: ~91s and AwaitAssert-times-out on a cold fixture, ~1s and passes warm. Baselined against a stashed tree to confirm it is not a LocalDb regression."
|
||||
}
|
||||
],
|
||||
"lastUpdated": "2026-07-19",
|
||||
"phase1Status": "COMPLETE - all 15 tasks done, live gate PASS. Branch feat/localdb-phase1 NOT merged/pushed."
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
# LocalDb Phase 2 — Gate Document
|
||||
|
||||
> **Status: NOT STARTED.** This is the gate, not the plan. Phase 2 work must not begin
|
||||
> until an implementation plan is written against the facts below and the open questions
|
||||
> in §5 are answered.
|
||||
|
||||
**Phase 1 live gate: PASS** (2026-07-19, docker rig). See the Task 12 commit for evidence.
|
||||
|
||||
**Phase 2 goal (from the design doc, scadaproj `docs/plans/2026-07-19-scadabridge-localdb-design.md` §1):**
|
||||
move `scadabridge.db`'s config tables and StoreAndForward's `sf_messages` into the same
|
||||
consolidated LocalDb file, then **delete** `SiteReplicationActor` and StoreAndForward's
|
||||
`ReplicationService` outright. CDC triggers replace hand-shipped ops; the library's
|
||||
snapshot resync replaces the bespoke chunked anti-entropy. The design explicitly chose
|
||||
**replace + delete, no dual-mechanism period** — recoverable from git.
|
||||
|
||||
---
|
||||
|
||||
## 1. What Phase 1 actually established
|
||||
|
||||
Facts a Phase 2 plan can rely on, all verified rather than assumed:
|
||||
|
||||
- One `ILocalDb` per site process at `LocalDb:Path`, **required** (`ValidateOnStart`).
|
||||
A site config missing it fails to boot.
|
||||
- `SiteLocalDbSetup.OnReady` is the single place tables are created and registered.
|
||||
Ordering is load-bearing: **DDL → `RegisterReplicated` → writes**, then the legacy
|
||||
migrator. Rows written before registration are invisible to the peer forever, silently.
|
||||
- Replication is registered in `SiteServiceRegistration` (not `Program.cs`) so the
|
||||
composition-root tests cover it; only `MapZbLocalDbSync` lives in `Program.cs`.
|
||||
- Inbound auth is `LocalDbSyncAuthInterceptor`, fail-closed, scoped to
|
||||
`/localdb_sync.v1.LocalDbSync/`, sharing the existing 8083 h2c listener.
|
||||
- Default-OFF is real and proven side-by-side: rig site-a replicates, site-b/site-c do not.
|
||||
- `ZbTelemetryOptions.Meters` is a **silent allowlist** (`SiteServiceRegistration.ObservedMeters`).
|
||||
- Bidirectional sync works with only one node configured as initiator.
|
||||
|
||||
## 2. What Phase 2 must port before deleting anything
|
||||
|
||||
The bespoke replicator's tests encode behaviour that the design's CDC replacement must
|
||||
still satisfy. **Port the test intents first, delete second** — the plan should treat the
|
||||
existing tests as the specification, not as code to be removed alongside the implementation.
|
||||
|
||||
- `tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/SiteReplicationActorTests.cs`
|
||||
- `tests/ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests/ReplicationServiceTests.cs`
|
||||
|
||||
Implementation under deletion:
|
||||
|
||||
- `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/SiteReplicationActor.cs`
|
||||
- `src/ZB.MOM.WW.ScadaBridge.StoreAndForward/ReplicationService.cs`
|
||||
|
||||
Known accepted behaviour to carry forward, not silently drop: the **N5 bounded-duplicate
|
||||
race** the bespoke replicator documents and accepts. A Phase 2 plan must state whether CDC
|
||||
inherits the same bound, a tighter one, or a different failure shape.
|
||||
|
||||
## 3. Substantially harder than Phase 1
|
||||
|
||||
Phase 1 moved two tables that **nothing replicated before**, so the worst case was "no
|
||||
worse than today". Phase 2 replaces a working mechanism, which makes it a different risk
|
||||
class:
|
||||
|
||||
- **`sf_messages` is an outbound buffer with side effects.** Phase 1's tables are
|
||||
read-mostly records; a store-and-forward row that resurrects or double-delivers sends
|
||||
real traffic to an external system. Park/requeue/remove becoming ordinary captured
|
||||
row updates/deletes needs its own analysis of what LWW does to an in-flight send.
|
||||
- **Config tables drive deployment.** A converged-but-wrong config row is a deployed
|
||||
instance behaving incorrectly, not a missing history entry.
|
||||
- **No dual-mechanism period.** The chosen posture means the cutover is the test. That
|
||||
raises the bar on the offline convergence suite and the live gate considerably.
|
||||
- **Migration is not a no-op this time.** Phase 1's migrator usually finds nothing,
|
||||
because the legacy files sat outside the volume. `scadabridge.db` and
|
||||
`store-and-forward.db` are real, populated, on the volume, and actively replicated
|
||||
while the migration runs.
|
||||
|
||||
## 4. Required inputs before writing the plan
|
||||
|
||||
1. The design doc §1 Phase 2 and §2 schema/migration sections, re-read against
|
||||
post-Phase-1 reality (several Phase 1 assumptions changed during execution — see the
|
||||
`amendments` array in `2026-07-19-localdb-adoption-phase1.md.tasks.json`).
|
||||
2. The two test files in §2, read as specifications.
|
||||
3. **Rig soak observations that do not exist yet** — see §5.
|
||||
|
||||
## 5. Open questions — answer these first
|
||||
|
||||
- **Oplog growth and churn under real config/S&F write rates.** Phase 1's tables are
|
||||
low-volume; `sf_messages` is not. `MaxOplogRows` / `MaxOplogAge` / snapshot-resync
|
||||
thresholds cannot be chosen from first principles. *This needs a Phase-1 rig soak that
|
||||
has not been run.* The live gate proved correctness, not steady-state behaviour over
|
||||
time.
|
||||
- **LWW semantics for in-flight store-and-forward sends.** What does last-writer-wins do
|
||||
when one node parks a message the other is mid-delivery on?
|
||||
- **Migration-under-load.** How does the one-time copy of an actively-replicating
|
||||
`scadabridge.db` interact with the bespoke replicator still running during cutover?
|
||||
- **Rollback story.** With no dual-mechanism period, what is the recovery path if the
|
||||
cutover fails in production — beyond "revert the commit"?
|
||||
- **Does the consolidated file stay appropriate?** One-DB-per-process was chosen partly
|
||||
because Phase 1's tables were small. Adding config + S&F changes the size and write
|
||||
profile of that single file.
|
||||
|
||||
## 6. Not in scope (unchanged from the design)
|
||||
|
||||
`auditlog.db` (diverges per node by design — central pulls the union; replicating it would
|
||||
double-forward), the secrets store (has its own answer in `Secrets.Replicator.SqlServer`
|
||||
hub mode), and central nodes (SQL-Server-first, no LocalDb use case).
|
||||
@@ -73,6 +73,37 @@ public record SiteHealthReport(
|
||||
/// indicates a stuck/blocked script holding a dedicated thread.
|
||||
/// </summary>
|
||||
public double? ScriptOldestBusyAgeSeconds { get; init; }
|
||||
|
||||
// LocalDb 2-node replication of the consolidated site database (Phase 1).
|
||||
// Additive init properties for the same reason as the scheduler gauges above:
|
||||
// the positional constructor stays untouched. Refreshed on the site by
|
||||
// LocalDbReplicationStatusReporter.
|
||||
|
||||
/// <summary>
|
||||
/// Whether a replication sync session is currently running with the peer site node,
|
||||
/// or <see langword="null"/> when replication is not wired on this node.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Nullable on purpose, and <see langword="false"/> is NOT the same as null.
|
||||
/// Replication ships default-OFF, so a site node with no peer configured reports
|
||||
/// <see langword="false"/> — that is a healthy, expected state, not an outage. Only
|
||||
/// a node that was configured with a peer and is reporting <see langword="false"/>
|
||||
/// is degraded, and distinguishing those two cases is the operator's job, not this
|
||||
/// field's.
|
||||
/// </remarks>
|
||||
public bool? LocalDbReplicationConnected { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Unacked replication oplog entries waiting to reach the peer, or
|
||||
/// <see langword="null"/> when unknown.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Null means unknown, never zero.</b> The underlying provider returns null when
|
||||
/// no backlog source is wired or the poll failed, and that must survive to the
|
||||
/// operator: a failed read rendered as "0 backlog" would report a broken replication
|
||||
/// pair as perfectly healthy. Collapsing null to 0 anywhere on this path is a bug.
|
||||
/// </remarks>
|
||||
public long? LocalDbOplogBacklog { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -141,6 +141,25 @@ public interface ISiteHealthCollector
|
||||
// SiteHealthCollector overrides this with the Interlocked.Exchange store.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replace the latest LocalDb replication status (peer-session connectivity and
|
||||
/// unacked oplog backlog) used by the next <see cref="CollectReport"/> call.
|
||||
/// Refreshed periodically by the <c>LocalDbReplicationStatusReporter</c> hosted
|
||||
/// service. Point-in-time: values are NOT reset on <see cref="CollectReport"/>.
|
||||
/// </summary>
|
||||
/// <param name="connected">
|
||||
/// Whether a sync session is currently running. <see langword="false"/> is the
|
||||
/// normal state on a node with no peer configured — replication ships default-OFF.
|
||||
/// </param>
|
||||
/// <param name="oplogBacklog">
|
||||
/// Unacked oplog entries, or <see langword="null"/> when unknown. Pass null through
|
||||
/// unchanged: a failed poll rendered as 0 would report a broken pair as healthy.
|
||||
/// </param>
|
||||
void SetLocalDbReplicationStatus(bool connected, long? oplogBacklog)
|
||||
{
|
||||
// Default no-op so test fakes do not need to be updated.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replace the latest script-execution-scheduler gauges (queue depth, busy
|
||||
/// thread count, and age in seconds of the oldest in-flight script) used by
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
|
||||
/// <summary>
|
||||
/// Site-side hosted service that periodically reads LocalDb replication status and pushes
|
||||
/// it into <see cref="ISiteHealthCollector"/>, so the next
|
||||
/// <see cref="ISiteHealthCollector.CollectReport"/> emits fresh
|
||||
/// <c>LocalDbReplicationConnected</c> / <c>LocalDbOplogBacklog</c> fields.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why delegates and not <c>ISyncStatus</c> directly.</b> Same reasoning as
|
||||
/// <see cref="SiteEventLogFailureCountReporter"/>: HealthMonitoring does not take a
|
||||
/// reference on the replication library. The Host site wiring captures the two reads as
|
||||
/// lambdas at registration time; this service only moves numbers.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Null backlog is preserved, never coerced to zero.</b> <c>ISyncStatus.OplogBacklog</c>
|
||||
/// is nullable precisely so a failed poll reads as "unknown" — rendering it as 0 would
|
||||
/// report a replication pair that cannot read its own oplog as perfectly healthy.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Cadence.</b> 30 s, matching <see cref="SiteEventLogFailureCountReporter"/> and
|
||||
/// <c>SiteAuditBacklogReporter</c>. Any exception during a probe is logged and swallowed;
|
||||
/// the next tick retries.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class LocalDbReplicationStatusReporter : IHostedService, IDisposable
|
||||
{
|
||||
/// <summary>Default poll cadence, matching the other site health bridges.</summary>
|
||||
internal static readonly TimeSpan DefaultRefreshInterval = TimeSpan.FromSeconds(30);
|
||||
|
||||
private readonly Func<bool> _connectedProvider;
|
||||
private readonly Func<long?> _oplogBacklogProvider;
|
||||
private readonly ISiteHealthCollector _collector;
|
||||
private readonly ILogger<LocalDbReplicationStatusReporter> _logger;
|
||||
private readonly TimeSpan _refreshInterval;
|
||||
private CancellationTokenSource? _cts;
|
||||
private Task? _loop;
|
||||
|
||||
/// <summary>Initializes a new instance of <see cref="LocalDbReplicationStatusReporter"/>.</summary>
|
||||
/// <param name="connectedProvider">Reads whether a peer sync session is currently running.</param>
|
||||
/// <param name="oplogBacklogProvider">Reads the unacked oplog backlog, or null when unknown.</param>
|
||||
/// <param name="collector">The site health collector receiving the snapshot.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
/// <param name="refreshInterval">Poll interval override; defaults to 30 s.</param>
|
||||
public LocalDbReplicationStatusReporter(
|
||||
Func<bool> connectedProvider,
|
||||
Func<long?> oplogBacklogProvider,
|
||||
ISiteHealthCollector collector,
|
||||
ILogger<LocalDbReplicationStatusReporter> logger,
|
||||
TimeSpan? refreshInterval = null)
|
||||
{
|
||||
_connectedProvider = connectedProvider ?? throw new ArgumentNullException(nameof(connectedProvider));
|
||||
_oplogBacklogProvider = oplogBacklogProvider ?? throw new ArgumentNullException(nameof(oplogBacklogProvider));
|
||||
_collector = collector ?? throw new ArgumentNullException(nameof(collector));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_refreshInterval = refreshInterval ?? DefaultRefreshInterval;
|
||||
}
|
||||
|
||||
/// <summary>Starts the polling loop, probing once immediately before entering the timed cycle.</summary>
|
||||
/// <param name="ct">Cancellation token signalling host shutdown.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task StartAsync(CancellationToken ct)
|
||||
{
|
||||
var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
_cts = cts;
|
||||
_loop = RunAsync(cts.Token);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Stops the polling loop.</summary>
|
||||
/// <param name="ct">Cancellation token for the stop operation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task StopAsync(CancellationToken ct)
|
||||
{
|
||||
if (_cts is null) return;
|
||||
|
||||
await _cts.CancelAsync().ConfigureAwait(false);
|
||||
if (_loop is not null)
|
||||
{
|
||||
// Await the loop so the reporter is quiescent before the host disposes the
|
||||
// collector out from under it.
|
||||
try { await _loop.ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) { /* expected on shutdown */ }
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
Probe();
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(_refreshInterval, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads both providers and pushes one snapshot onto the collector; never throws.
|
||||
/// Public so a test can drive a single deterministic probe instead of starting the
|
||||
/// service and waiting out the 30 s cadence.
|
||||
/// </summary>
|
||||
public void Probe()
|
||||
{
|
||||
try
|
||||
{
|
||||
_collector.SetLocalDbReplicationStatus(
|
||||
_connectedProvider(), _oplogBacklogProvider());
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Health reporting must never take the site node down. The previous snapshot
|
||||
// stays on the collector and the next tick retries.
|
||||
_logger.LogWarning(ex, "Failed to read LocalDb replication status for the site health report.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose() => _cts?.Dispose();
|
||||
}
|
||||
@@ -21,6 +21,12 @@ public static class ServiceCollectionExtensions
|
||||
/// </summary>
|
||||
private sealed class SiteEventLogHealthMetricsBridgeMarker { }
|
||||
|
||||
/// <summary>
|
||||
/// Sentinel marker for <see cref="AddLocalDbReplicationHealthBridge"/>'s idempotency
|
||||
/// guard — same rationale as <see cref="SiteEventLogHealthMetricsBridgeMarker"/>.
|
||||
/// </summary>
|
||||
private sealed class LocalDbReplicationHealthBridgeMarker { }
|
||||
|
||||
/// <summary>
|
||||
/// Register site-side health monitoring services (metric collection + periodic reporting).
|
||||
/// Call this on site nodes only. For central, call AddCentralHealthAggregation() instead.
|
||||
@@ -153,6 +159,46 @@ public static class ServiceCollectionExtensions
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Bridge LocalDb replication status (peer connectivity + unacked oplog backlog) onto
|
||||
/// the site health report. Must be called AFTER <c>AddSiteHealthMonitoring</c>
|
||||
/// (registers <see cref="ISiteHealthCollector"/>) and after the replication engine is
|
||||
/// registered. Idempotent via a marker sentinel.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Delegates rather than an <c>ISyncStatus</c> parameter keep HealthMonitoring free of
|
||||
/// a reference on the replication library, matching
|
||||
/// <see cref="AddSiteEventLogHealthMetricsBridge"/>. Pass the backlog through as
|
||||
/// nullable — coercing null to 0 would report an unreadable oplog as a healthy one.
|
||||
/// </remarks>
|
||||
/// <param name="services">The service collection to register into.</param>
|
||||
/// <param name="connectedProvider">Given the root provider, returns a reader for "a sync session is running".</param>
|
||||
/// <param name="oplogBacklogProvider">Given the root provider, returns a reader for the unacked backlog (null = unknown).</param>
|
||||
/// <returns>The same <see cref="IServiceCollection"/> for chaining.</returns>
|
||||
public static IServiceCollection AddLocalDbReplicationHealthBridge(
|
||||
this IServiceCollection services,
|
||||
Func<IServiceProvider, Func<bool>> connectedProvider,
|
||||
Func<IServiceProvider, Func<long?>> oplogBacklogProvider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(connectedProvider);
|
||||
ArgumentNullException.ThrowIfNull(oplogBacklogProvider);
|
||||
|
||||
if (services.Any(d => d.ServiceType == typeof(LocalDbReplicationHealthBridgeMarker)))
|
||||
{
|
||||
return services;
|
||||
}
|
||||
|
||||
services.AddSingleton<LocalDbReplicationHealthBridgeMarker>();
|
||||
services.AddHostedService(sp => new LocalDbReplicationStatusReporter(
|
||||
connectedProvider(sp),
|
||||
oplogBacklogProvider(sp),
|
||||
sp.GetRequiredService<ISiteHealthCollector>(),
|
||||
sp.GetRequiredService<ILogger<LocalDbReplicationStatusReporter>>()));
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Register the <see cref="HealthMonitoringOptionsValidator"/>
|
||||
/// so a misconfigured <c>ScadaBridge:HealthMonitoring</c> section (zero/negative
|
||||
|
||||
@@ -18,6 +18,10 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
private int _auditRedactionFailures;
|
||||
private volatile SiteAuditBacklogSnapshot? _siteAuditBacklog;
|
||||
private long _siteEventLogWriteFailures;
|
||||
// One volatile tuple rather than two independent fields: a torn read that paired a
|
||||
// fresh Connected with a stale backlog would be indistinguishable from a real state.
|
||||
// Null = the reporter has not yet run (or replication is not wired).
|
||||
private volatile Tuple<bool, long?>? _localDbReplicationStatus;
|
||||
private readonly ConcurrentDictionary<string, ConnectionHealth> _connectionStatuses = new();
|
||||
private readonly ConcurrentDictionary<string, TagResolutionStatus> _tagResolutionCounts = new();
|
||||
private readonly ConcurrentDictionary<string, string> _connectionEndpoints = new();
|
||||
@@ -94,6 +98,12 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
Interlocked.Exchange(ref _siteEventLogWriteFailures, count);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetLocalDbReplicationStatus(bool connected, long? oplogBacklog)
|
||||
{
|
||||
_localDbReplicationStatus = Tuple.Create(connected, oplogBacklog);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health)
|
||||
{
|
||||
@@ -219,6 +229,9 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
var deadLetters = Interlocked.Exchange(ref _deadLetterCount, 0);
|
||||
var siteAuditWriteFailures = Interlocked.Exchange(ref _siteAuditWriteFailures, 0);
|
||||
var auditRedactionFailures = Interlocked.Exchange(ref _auditRedactionFailures, 0);
|
||||
// Single read of the volatile tuple — two reads could straddle a reporter tick and
|
||||
// pair a fresh Connected with a stale backlog.
|
||||
var localDbReplication = _localDbReplicationStatus;
|
||||
|
||||
// Snapshot current connection and tag resolution state
|
||||
var connectionStatuses = new Dictionary<string, ConnectionHealth>(_connectionStatuses);
|
||||
@@ -259,7 +272,12 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
{
|
||||
ScriptQueueDepth = Interlocked.CompareExchange(ref _scriptQueueDepth, 0, 0),
|
||||
ScriptBusyThreads = Interlocked.CompareExchange(ref _scriptBusyThreads, 0, 0),
|
||||
ScriptOldestBusyAgeSeconds = ReadScriptOldestBusyAgeSeconds()
|
||||
ScriptOldestBusyAgeSeconds = ReadScriptOldestBusyAgeSeconds(),
|
||||
// Both fields come from the ONE snapshot read above. Null (the reporter has
|
||||
// not run) leaves both report fields null — "no data", not "disconnected
|
||||
// with an empty backlog".
|
||||
LocalDbReplicationConnected = localDbReplication?.Item1,
|
||||
LocalDbOplogBacklog = localDbReplication?.Item2
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Grpc.Core;
|
||||
using Grpc.Core.Interceptors;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.LocalDb.Replication;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
|
||||
/// <summary>
|
||||
/// Gates the LocalDb passive sync endpoint. The replication library deliberately leaves
|
||||
/// inbound authentication to the host — its <c>LocalDbSyncService</c> verifies nothing —
|
||||
/// so without this interceptor anything that can reach the site node's gRPC port could
|
||||
/// stream arbitrary rows into the consolidated site database.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Scoped by method path.</b> Only calls under
|
||||
/// <c>/localdb_sync.v1.LocalDbSync/</c> are gated; every other method — notably the
|
||||
/// existing <c>SiteStream</c> service sharing this listener — passes through untouched.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Fail-closed.</b> With no <c>LocalDb:Replication:ApiKey</c> configured, NO sync
|
||||
/// stream is accepted, authenticated or not. That is the deliberate choice: the
|
||||
/// alternative — treating "no key" as "no auth required" — would silently expose the
|
||||
/// endpoint on exactly the default configuration every site node ships with. An operator
|
||||
/// enabling replication must set the same key on both nodes, which is already required
|
||||
/// for the initiator to dial out (<c>SyncBackgroundService</c> sends
|
||||
/// <c>Authorization: Bearer <key></c>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Comparison is <see cref="CryptographicOperations.FixedTimeEquals"/> over UTF-8 bytes,
|
||||
/// so a wrong key cannot be recovered byte-by-byte from response timing. Length differences
|
||||
/// are unavoidably observable and are not sensitive.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class LocalDbSyncAuthInterceptor : Interceptor
|
||||
{
|
||||
private const string ServicePrefix = "/localdb_sync.v1.LocalDbSync/";
|
||||
private const string AuthorizationHeader = "authorization";
|
||||
private const string BearerPrefix = "Bearer ";
|
||||
|
||||
private readonly IOptions<ReplicationOptions> _options;
|
||||
private readonly ILogger<LocalDbSyncAuthInterceptor> _logger;
|
||||
|
||||
/// <summary>Creates the interceptor.</summary>
|
||||
/// <param name="options">Replication options; <c>ApiKey</c> is the expected bearer token.</param>
|
||||
/// <param name="logger">Logger for denial diagnostics.</param>
|
||||
public LocalDbSyncAuthInterceptor(
|
||||
IOptions<ReplicationOptions> options,
|
||||
ILogger<LocalDbSyncAuthInterceptor> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<TResponse> UnaryServerHandler<TRequest, TResponse>(
|
||||
TRequest request,
|
||||
ServerCallContext context,
|
||||
UnaryServerMethod<TRequest, TResponse> continuation)
|
||||
{
|
||||
Authorize(context);
|
||||
return continuation(request, context);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task DuplexStreamingServerHandler<TRequest, TResponse>(
|
||||
IAsyncStreamReader<TRequest> requestStream,
|
||||
IServerStreamWriter<TResponse> responseStream,
|
||||
ServerCallContext context,
|
||||
DuplexStreamingServerMethod<TRequest, TResponse> continuation)
|
||||
{
|
||||
Authorize(context);
|
||||
return continuation(requestStream, responseStream, context);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<TResponse> ClientStreamingServerHandler<TRequest, TResponse>(
|
||||
IAsyncStreamReader<TRequest> requestStream,
|
||||
ServerCallContext context,
|
||||
ClientStreamingServerMethod<TRequest, TResponse> continuation)
|
||||
{
|
||||
Authorize(context);
|
||||
return continuation(requestStream, context);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task ServerStreamingServerHandler<TRequest, TResponse>(
|
||||
TRequest request,
|
||||
IServerStreamWriter<TResponse> responseStream,
|
||||
ServerCallContext context,
|
||||
ServerStreamingServerMethod<TRequest, TResponse> continuation)
|
||||
{
|
||||
Authorize(context);
|
||||
return continuation(request, responseStream, context);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Throws <see cref="RpcException"/> with <see cref="StatusCode.PermissionDenied"/> if
|
||||
/// this is a sync call that does not carry the configured bearer token. Non-sync calls
|
||||
/// return immediately.
|
||||
/// </summary>
|
||||
private void Authorize(ServerCallContext context)
|
||||
{
|
||||
if (!context.Method.StartsWith(ServicePrefix, StringComparison.Ordinal))
|
||||
return;
|
||||
|
||||
var expected = _options.Value.ApiKey;
|
||||
if (string.IsNullOrEmpty(expected))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Rejected a LocalDb sync call to {Method}: no LocalDb:Replication:ApiKey is configured, " +
|
||||
"so the passive sync endpoint is closed. Configure the same key on both nodes of the pair.",
|
||||
context.Method);
|
||||
throw new RpcException(new Status(
|
||||
StatusCode.PermissionDenied,
|
||||
"LocalDb sync is not accepting connections: no API key is configured on this node."));
|
||||
}
|
||||
|
||||
var presented = ExtractBearerToken(context.RequestHeaders);
|
||||
if (presented is null || !FixedTimeEquals(presented, expected))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Rejected a LocalDb sync call to {Method}: {Reason}.",
|
||||
context.Method,
|
||||
presented is null ? "no bearer token presented" : "bearer token did not match");
|
||||
throw new RpcException(new Status(
|
||||
StatusCode.PermissionDenied,
|
||||
"LocalDb sync authentication failed."));
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ExtractBearerToken(Metadata headers)
|
||||
{
|
||||
// gRPC lowercases header keys on the wire; compare case-insensitively anyway so a
|
||||
// hand-built Metadata in a test behaves the same as a real request.
|
||||
foreach (var entry in headers)
|
||||
{
|
||||
if (!string.Equals(entry.Key, AuthorizationHeader, StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
var value = entry.Value;
|
||||
if (value is not null && value.StartsWith(BearerPrefix, StringComparison.OrdinalIgnoreCase))
|
||||
return value[BearerPrefix.Length..];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool FixedTimeEquals(string presented, string expected)
|
||||
=> CryptographicOperations.FixedTimeEquals(
|
||||
Encoding.UTF8.GetBytes(presented), Encoding.UTF8.GetBytes(expected));
|
||||
}
|
||||
@@ -7,6 +7,7 @@ using ZB.MOM.WW.Auth.AspNetCore;
|
||||
using ZB.MOM.WW.Health;
|
||||
using ZB.MOM.WW.Health.Akka;
|
||||
using ZB.MOM.WW.Health.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.LocalDb.Replication;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog;
|
||||
using ZB.MOM.WW.ScadaBridge.CentralUI;
|
||||
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
|
||||
@@ -513,10 +514,15 @@ try
|
||||
});
|
||||
|
||||
// gRPC server registration
|
||||
builder.Services.AddGrpc();
|
||||
// The interceptor gates ONLY /localdb_sync.v1.LocalDbSync/ — SiteStream calls on
|
||||
// this same pipeline pass through untouched. It is fail-closed: with no
|
||||
// LocalDb:Replication:ApiKey configured, no sync stream is accepted at all.
|
||||
builder.Services.AddGrpc(options =>
|
||||
options.Interceptors.Add<LocalDbSyncAuthInterceptor>());
|
||||
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
|
||||
|
||||
// Existing site service registrations
|
||||
// Existing site service registrations (this is also where LocalDb and its
|
||||
// replication engine are registered — see SiteServiceRegistration)
|
||||
SiteServiceRegistration.Configure(builder.Services, builder.Configuration);
|
||||
|
||||
var app = builder.Build();
|
||||
@@ -535,6 +541,13 @@ try
|
||||
// Map gRPC service — resolves the singleton SiteStreamGrpcServer from DI
|
||||
app.MapGrpcService<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
|
||||
|
||||
// The passive half of LocalDb replication: the peer node dials THIS endpoint.
|
||||
// It shares the Kestrel h2c listener the site gRPC server already uses, so no
|
||||
// listener or port changes are needed. Mapping it is harmless with no peer
|
||||
// configured — nothing dials it, and LocalDbSyncAuthInterceptor (registered on
|
||||
// AddGrpc above) rejects anything that tries until an ApiKey is configured.
|
||||
app.MapZbLocalDbSync();
|
||||
|
||||
// Site-shutdown ordering. ApplicationStopping
|
||||
// fires BEFORE IHostedService.StopAsync runs, so the gRPC server
|
||||
// refuses new streams (Unavailable) and cancels every active stream
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
|
||||
/// <summary>
|
||||
/// One-time copy of the pre-Phase-1 site databases (<c>site-tracking.db</c> and
|
||||
/// <c>site_events.db</c>) into the consolidated <c>ZB.MOM.WW.LocalDb</c> database.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Runs after <c>RegisterReplicated</c>, deliberately.</b> Capture is trigger-based, so
|
||||
/// rows inserted before registration would never enter the oplog and would never reach the
|
||||
/// peer. Migrating after registration means the migrated history replicates like any other
|
||||
/// write — which is the entire point of Phase 1.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Idempotent twice over.</b> Tracking rows carry the same TEXT primary key they always
|
||||
/// had, so <c>INSERT OR IGNORE</c> is naturally idempotent. Legacy events had autoincrement
|
||||
/// integer ids, which the consolidated schema replaced with GUIDs; they are given
|
||||
/// <i>deterministic</i> ids of the form <c>mig-{NodeName}-{legacyId}</c> rather than fresh
|
||||
/// GUIDs, so a migration that crashes and re-runs cannot duplicate events. The NodeName
|
||||
/// prefix keeps the two nodes of a pair from colliding on their independent legacy id
|
||||
/// sequences.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>All-or-nothing.</b> The copy runs in one transaction and the legacy file is renamed
|
||||
/// to <c><name>.migrated</c> only after commit. A failure throws out of
|
||||
/// <c>OnReady</c>, which fails host startup with the legacy files untouched — no
|
||||
/// half-migrated state to reason about. A second boot sees the renamed file and no-ops.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Finding nothing is the expected case on the docker rig.</b> Neither legacy config key
|
||||
/// is set in any rig appsettings, so both fall back to CWD-relative code defaults
|
||||
/// (<c>/app/site-tracking.db</c>, <c>/app/site_events.db</c>) that sit OUTSIDE the mounted
|
||||
/// data volume — meaning they were already being discarded on every container recreate.
|
||||
/// Phase 1 incidentally fixes that data-loss bug by consolidating into
|
||||
/// <c>/app/data/site-localdb.db</c>. A no-op here is a legitimate result, not a failure.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class SiteLocalDbLegacyMigrator
|
||||
{
|
||||
private const string MigratedSuffix = ".migrated";
|
||||
|
||||
/// <summary>Default legacy tracking connection string (<c>OperationTrackingOptions.ConnectionString</c>).</summary>
|
||||
private const string DefaultTrackingConnectionString = "Data Source=site-tracking.db";
|
||||
|
||||
/// <summary>Default legacy event-log path (<c>SiteEventLogOptions.DatabasePath</c>).</summary>
|
||||
private const string DefaultEventLogPath = "site_events.db";
|
||||
|
||||
/// <summary>
|
||||
/// Copies any legacy site databases into <paramref name="db"/>, then renames them.
|
||||
/// </summary>
|
||||
/// <param name="db">The consolidated site database, with both tables already registered.</param>
|
||||
/// <param name="config">Configuration supplying the legacy paths and the node name.</param>
|
||||
public static void Migrate(ILocalDb db, IConfiguration config)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(db);
|
||||
ArgumentNullException.ThrowIfNull(config);
|
||||
|
||||
var nodeName = config["ScadaBridge:Node:NodeName"] ?? "unknown-node";
|
||||
|
||||
MigrateTracking(db, ResolveTrackingPath(config));
|
||||
MigrateEvents(db, ResolveEventLogPath(config), nodeName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the legacy tracking database path from the OLD connection-string key,
|
||||
/// falling back to the code default. Relative paths resolve against the process working
|
||||
/// directory — NOT the data volume — because that is where the old code actually put them.
|
||||
/// </summary>
|
||||
internal static string ResolveTrackingPath(IConfiguration config)
|
||||
{
|
||||
var connectionString =
|
||||
config["ScadaBridge:OperationTracking:ConnectionString"] ?? DefaultTrackingConnectionString;
|
||||
|
||||
string dataSource;
|
||||
try
|
||||
{
|
||||
dataSource = new SqliteConnectionStringBuilder(connectionString).DataSource;
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// An unparseable legacy connection string means there is nothing to migrate
|
||||
// from. Do not fail the host over a stale key we are about to stop reading.
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
// In-memory legacy databases (test/dev configurations) have nothing durable to
|
||||
// migrate, and Path.GetFullPath on them would produce nonsense.
|
||||
if (string.IsNullOrWhiteSpace(dataSource) ||
|
||||
dataSource.Equals(":memory:", StringComparison.OrdinalIgnoreCase) ||
|
||||
dataSource.StartsWith("file:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return Path.GetFullPath(dataSource);
|
||||
}
|
||||
|
||||
/// <summary>Resolves the legacy event-log path from the OLD key, falling back to the code default.</summary>
|
||||
internal static string ResolveEventLogPath(IConfiguration config)
|
||||
{
|
||||
var path = config["ScadaBridge:SiteEventLog:DatabasePath"] ?? DefaultEventLogPath;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(path) ||
|
||||
path.Equals(":memory:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return Path.GetFullPath(path);
|
||||
}
|
||||
|
||||
private static void MigrateTracking(ILocalDb db, string legacyPath)
|
||||
{
|
||||
if (!ShouldMigrate(legacyPath)) return;
|
||||
|
||||
var rows = ReadAll(
|
||||
legacyPath,
|
||||
"""
|
||||
SELECT TrackedOperationId, Kind, TargetSummary, Status,
|
||||
RetryCount, LastError, HttpStatus,
|
||||
CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc,
|
||||
SourceInstanceId, SourceScript, SourceNode
|
||||
FROM OperationTracking;
|
||||
""",
|
||||
expectedColumns: 13);
|
||||
|
||||
// A legacy file with no OperationTracking table at all reads as "nothing to do";
|
||||
// it still gets renamed so the probe does not repeat on every boot.
|
||||
if (rows is not null)
|
||||
{
|
||||
using var connection = db.CreateConnection();
|
||||
using var transaction = connection.BeginTransaction();
|
||||
|
||||
foreach (var row in rows)
|
||||
{
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.Transaction = transaction;
|
||||
cmd.CommandText = """
|
||||
INSERT OR IGNORE INTO OperationTracking (
|
||||
TrackedOperationId, Kind, TargetSummary, Status,
|
||||
RetryCount, LastError, HttpStatus,
|
||||
CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc,
|
||||
SourceInstanceId, SourceScript, SourceNode
|
||||
) VALUES (
|
||||
$p0, $p1, $p2, $p3, $p4, $p5, $p6,
|
||||
$p7, $p8, $p9, $p10, $p11, $p12
|
||||
);
|
||||
""";
|
||||
Bind(cmd, row);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
|
||||
MarkMigrated(legacyPath);
|
||||
}
|
||||
|
||||
private static void MigrateEvents(ILocalDb db, string legacyPath, string nodeName)
|
||||
{
|
||||
if (!ShouldMigrate(legacyPath)) return;
|
||||
|
||||
var rows = ReadAll(
|
||||
legacyPath,
|
||||
"""
|
||||
SELECT id, timestamp, event_type, severity, instance_id, source, message, details
|
||||
FROM site_events;
|
||||
""",
|
||||
expectedColumns: 8);
|
||||
|
||||
if (rows is not null)
|
||||
{
|
||||
using var connection = db.CreateConnection();
|
||||
using var transaction = connection.BeginTransaction();
|
||||
|
||||
foreach (var row in rows)
|
||||
{
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.Transaction = transaction;
|
||||
cmd.CommandText = """
|
||||
INSERT OR IGNORE INTO site_events (
|
||||
id, timestamp, event_type, severity, instance_id, source, message, details
|
||||
) VALUES (
|
||||
$id, $p1, $p2, $p3, $p4, $p5, $p6, $p7
|
||||
);
|
||||
""";
|
||||
|
||||
// Deterministic id, NOT a fresh GUID: a crashed-then-rerun migration must
|
||||
// not duplicate events, and INSERT OR IGNORE can only dedupe on a stable key.
|
||||
cmd.Parameters.AddWithValue("$id", $"mig-{nodeName}-{row[0] ?? "null"}");
|
||||
for (var i = 1; i < row.Length; i++)
|
||||
cmd.Parameters.AddWithValue($"$p{i}", row[i] ?? (object)DBNull.Value);
|
||||
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
|
||||
MarkMigrated(legacyPath);
|
||||
}
|
||||
|
||||
/// <summary>True when there is a legacy file present that has not already been migrated.</summary>
|
||||
private static bool ShouldMigrate(string legacyPath)
|
||||
=> !string.IsNullOrEmpty(legacyPath)
|
||||
&& File.Exists(legacyPath)
|
||||
&& !File.Exists(legacyPath + MigratedSuffix);
|
||||
|
||||
/// <summary>
|
||||
/// Reads every row of <paramref name="sql"/> from the legacy file, or null when the
|
||||
/// table does not exist (an old file predating the table is not an error).
|
||||
/// </summary>
|
||||
private static List<object?[]>? ReadAll(string legacyPath, string sql, int expectedColumns)
|
||||
{
|
||||
using var connection = new SqliteConnection($"Data Source={legacyPath};Mode=ReadOnly");
|
||||
connection.Open();
|
||||
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = sql;
|
||||
|
||||
SqliteDataReader reader;
|
||||
try
|
||||
{
|
||||
reader = cmd.ExecuteReader();
|
||||
}
|
||||
catch (SqliteException)
|
||||
{
|
||||
// "no such table" / "no such column" — a legacy file from before this table
|
||||
// existed, or a shape we do not recognise. Nothing to copy.
|
||||
return null;
|
||||
}
|
||||
|
||||
var rows = new List<object?[]>();
|
||||
using (reader)
|
||||
{
|
||||
while (reader.Read())
|
||||
{
|
||||
var row = new object?[expectedColumns];
|
||||
for (var i = 0; i < expectedColumns; i++)
|
||||
row[i] = reader.IsDBNull(i) ? null : reader.GetValue(i);
|
||||
rows.Add(row);
|
||||
}
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
private static void Bind(SqliteCommand cmd, object?[] row)
|
||||
{
|
||||
for (var i = 0; i < row.Length; i++)
|
||||
cmd.Parameters.AddWithValue($"$p{i}", row[i] ?? (object)DBNull.Value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Renames the legacy file so a later boot skips it. Done only after the copy has
|
||||
/// committed; the file is kept rather than deleted so an operator can still inspect it.
|
||||
/// </summary>
|
||||
private static void MarkMigrated(string legacyPath)
|
||||
=> File.Move(legacyPath, legacyPath + MigratedSuffix);
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking;
|
||||
@@ -29,12 +30,15 @@ namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
public static class SiteLocalDbSetup
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates the site node's replicated tables and opts them into change capture.
|
||||
/// Creates the site node's replicated tables, opts them into change capture, and
|
||||
/// migrates any pre-Phase-1 databases in.
|
||||
/// </summary>
|
||||
/// <param name="db">The LocalDb instance being initialized.</param>
|
||||
public static void OnReady(ILocalDb db)
|
||||
/// <param name="config">Configuration, for the legacy migrator's old path keys and node name.</param>
|
||||
public static void OnReady(ILocalDb db, IConfiguration config)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(db);
|
||||
ArgumentNullException.ThrowIfNull(config);
|
||||
|
||||
using (var connection = db.CreateConnection())
|
||||
{
|
||||
@@ -47,5 +51,9 @@ public static class SiteLocalDbSetup
|
||||
// capture). Registration is idempotent and installs the capture triggers.
|
||||
db.RegisterReplicated("OperationTracking");
|
||||
db.RegisterReplicated("site_events");
|
||||
|
||||
// AFTER registration, so migrated rows enter the oplog and reach the peer like
|
||||
// any other write. Before it, they would be invisible to replication forever.
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.LocalDb.Replication;
|
||||
using ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
using ZB.MOM.WW.Telemetry;
|
||||
|
||||
@@ -26,6 +27,24 @@ namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
/// </summary>
|
||||
public static class SiteServiceRegistration
|
||||
{
|
||||
/// <summary>
|
||||
/// Every <see cref="System.Diagnostics.Metrics.Meter"/> OpenTelemetry is told to observe.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <c>ZbTelemetryOptions.Meters</c> is an ALLOWLIST, and an unlisted meter's instruments
|
||||
/// are simply never observed — no error, no warning, no missing-metric signal anywhere.
|
||||
/// The LocalDb replication meter was silently absent from the rig's <c>/metrics</c>
|
||||
/// scrape until the Phase 1 live gate went looking for it (the plan had assumed it
|
||||
/// flowed through automatically). Hoisted to a named constant so the list is
|
||||
/// assertable; adding a meter anywhere in the product means adding it here.
|
||||
/// </remarks>
|
||||
public static readonly string[] ObservedMeters =
|
||||
[
|
||||
ScadaBridgeTelemetry.MeterName,
|
||||
// Emitted only on site nodes; harmless on central, which never records against it.
|
||||
LocalDbMetrics.MeterName,
|
||||
];
|
||||
|
||||
/// <summary>Registers all DI services required for the site role.</summary>
|
||||
/// <param name="services">The service collection to register into.</param>
|
||||
/// <param name="config">Application configuration for options binding.</param>
|
||||
@@ -56,11 +75,21 @@ public static class SiteServiceRegistration
|
||||
//
|
||||
// Storage is registered UNCONDITIONALLY and needs no flag: with no peer
|
||||
// configured, LocalDb is simply a local SQLite file and the replication
|
||||
// initiator idles. Replication itself is opt-in via LocalDb:Replication:PeerAddress
|
||||
// and is wired in Program.cs, which owns the gRPC host the sync endpoint needs.
|
||||
// initiator idles.
|
||||
//
|
||||
// Design: scadaproj docs/plans/2026-07-19-scadabridge-localdb-design.md
|
||||
services.AddZbLocalDb(config, SiteLocalDbSetup.OnReady);
|
||||
services.AddZbLocalDb(config, db => SiteLocalDbSetup.OnReady(db, config));
|
||||
|
||||
// The replication engine, likewise unconditional but INERT by default: with no
|
||||
// LocalDb:Replication:PeerAddress the initiating SyncBackgroundService starts and
|
||||
// immediately idles, and nothing dials the passive endpoint. A site pair
|
||||
// replicates only once an operator sets a peer on both nodes.
|
||||
//
|
||||
// Registered HERE rather than in Program.cs (where the plan first put it) so the
|
||||
// composition-root tests, which build this graph and not Program.cs, actually
|
||||
// cover it. The endpoint half — MapZbLocalDbSync — has to stay in Program.cs
|
||||
// because it needs the WebApplication.
|
||||
services.AddZbLocalDbReplication(config);
|
||||
|
||||
services.AddDataConnectionLayer();
|
||||
// Local SQLite store by default; a local store replicating against a shared SQL-Server hub
|
||||
@@ -89,6 +118,16 @@ public static class SiteServiceRegistration
|
||||
services.AddSiteEventLogHealthMetricsBridge(
|
||||
sp => () => sp.GetRequiredService<ISiteEventLogger>().FailedWriteCount);
|
||||
|
||||
// LocalDb replication — bridge ISyncStatus onto the site health report. Registered
|
||||
// unconditionally alongside the engine: on a node with no peer this reports
|
||||
// Connected=false with a real 0 backlog, which is the healthy default-OFF state.
|
||||
// OplogBacklog is passed through NULLABLE on purpose — null means the poll failed
|
||||
// or no provider is wired, and flattening it to 0 would render a replication pair
|
||||
// that cannot read its own oplog as perfectly healthy.
|
||||
services.AddLocalDbReplicationHealthBridge(
|
||||
sp => () => sp.GetRequiredService<ISyncStatus>().Connected,
|
||||
sp => () => sp.GetRequiredService<ISyncStatus>().OplogBacklog);
|
||||
|
||||
// Audit Log — site-side hot-path writer + telemetry collaborators.
|
||||
// The SiteAuditTelemetryActor itself is registered by AkkaHostedService
|
||||
// in the site-role block; this call wires every DI dependency it (and
|
||||
@@ -241,7 +280,7 @@ public static class SiteServiceRegistration
|
||||
o.ServiceName = "scadabridge";
|
||||
o.SiteId = config["ScadaBridge:Node:SiteId"] ?? "central";
|
||||
o.NodeRole = config["ScadaBridge:Node:Role"];
|
||||
o.Meters = [ScadaBridgeTelemetry.MeterName];
|
||||
o.Meters = ObservedMeters;
|
||||
if (Enum.TryParse<ZbExporter>(config["ScadaBridge:Telemetry:Exporter"], ignoreCase: true, out var exporter))
|
||||
o.Exporter = exporter;
|
||||
var otlp = config["ScadaBridge:Telemetry:OtlpEndpoint"];
|
||||
|
||||
@@ -6,7 +6,14 @@ public class SiteEventLogOptions
|
||||
public int RetentionDays { get; set; } = 30;
|
||||
/// <summary>Maximum SQLite database size in megabytes before old entries are purged; default 1024 MB.</summary>
|
||||
public int MaxStorageMb { get; set; } = 1024;
|
||||
/// <summary>File path for the site event log SQLite database.</summary>
|
||||
/// <summary>
|
||||
/// MIGRATION-ONLY as of LocalDb Phase 1. <see cref="SiteEventLogger"/> no longer reads
|
||||
/// this: <c>site_events</c> lives in the consolidated site database at
|
||||
/// <c>LocalDb:Path</c>. The only remaining consumer is
|
||||
/// <c>SiteLocalDbLegacyMigrator</c>, which uses it to locate a pre-Phase-1
|
||||
/// <c>site_events.db</c> and copy it in once. Safe to delete from a config once that
|
||||
/// node's migration has run.
|
||||
/// </summary>
|
||||
public string DatabasePath { get; set; } = "site_events.db";
|
||||
/// <summary>Maximum number of rows returned per paginated query; default 500.</summary>
|
||||
public int QueryPageSize { get; set; } = 500;
|
||||
|
||||
@@ -2,13 +2,16 @@ using System.Threading.Channels;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteEventLogging;
|
||||
|
||||
/// <summary>
|
||||
/// Records operational events to a local SQLite database.
|
||||
/// Only the active node generates events. Not replicated to standby.
|
||||
/// On failover, the new active node starts a fresh log.
|
||||
/// Records operational events into the consolidated site database (LocalDb Phase 1).
|
||||
/// Only the active node generates events, but <c>site_events</c> is a replicated table:
|
||||
/// when the pair has a peer configured, the standby holds the same history and a failover
|
||||
/// no longer starts a fresh log. With no peer configured the database is simply local —
|
||||
/// replication is opt-in via <c>LocalDb:Replication:PeerAddress</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
@@ -40,29 +43,36 @@ public class SiteEventLogger : ISiteEventLogger, IDisposable
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes the event logger, opens the SQLite connection, and starts the background writer loop.
|
||||
/// Initializes the event logger over the consolidated site database and starts the
|
||||
/// background writer loop.
|
||||
/// </summary>
|
||||
/// <param name="options">Site event log configuration (database path, retention settings).</param>
|
||||
/// <param name="options">Site event log configuration (retention settings, queue capacity).</param>
|
||||
/// <param name="logger">Logger for write-failure diagnostics.</param>
|
||||
/// <param name="connectionStringOverride">Optional connection string override; uses the configured path when null.</param>
|
||||
/// <param name="localDb">
|
||||
/// The consolidated site database. The connection it hands out is already open and
|
||||
/// carries the <c>zb_hlc_next()</c> UDF that <c>site_events</c>' capture triggers
|
||||
/// call. This logger owns the connection for its lifetime and disposes it.
|
||||
/// </param>
|
||||
/// <remarks>
|
||||
/// The former <c>connectionStringOverride</c> seam is deliberately gone rather than
|
||||
/// preserved. <c>site_events</c> is a replicated table, so a raw connection lacking
|
||||
/// the UDF would fail closed on every insert — an override would only let a caller
|
||||
/// build a logger that cannot write. <c>SiteEventLogOptions.DatabasePath</c> is
|
||||
/// likewise no longer read: the location is <c>LocalDb:Path</c>.
|
||||
/// </remarks>
|
||||
public SiteEventLogger(
|
||||
IOptions<SiteEventLogOptions> options,
|
||||
ILogger<SiteEventLogger> logger,
|
||||
string? connectionStringOverride = null)
|
||||
ILocalDb localDb)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(localDb);
|
||||
|
||||
_logger = logger;
|
||||
|
||||
// Cache=Shared is a cross-connection optimisation
|
||||
// that lets multiple SqliteConnections share an in-process page cache.
|
||||
// This logger owns exactly one SqliteConnection and serialises all
|
||||
// access through _writeLock, so the mode is dormant — at best dead
|
||||
// configuration, at worst a small future foot-gun for any second
|
||||
// connection opened to the same file. A test path that genuinely
|
||||
// needs Cache=Shared can still inject it via connectionStringOverride.
|
||||
var connectionString = connectionStringOverride
|
||||
?? $"Data Source={options.Value.DatabasePath}";
|
||||
_connection = new SqliteConnection(connectionString);
|
||||
_connection.Open();
|
||||
// Already open — CreateConnection returns a live, pragma-configured,
|
||||
// UDF-registered connection. Calling Open() on it again would throw.
|
||||
_connection = localDb.CreateConnection();
|
||||
|
||||
InitializeSchema();
|
||||
|
||||
|
||||
@@ -7,9 +7,12 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking;
|
||||
public class OperationTrackingOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Full ADO.NET connection string for the SQLite database (e.g.
|
||||
/// <c>"Data Source=site-tracking.db"</c>). Tests use the
|
||||
/// <c>Mode=Memory;Cache=Shared</c> form to keep the database in-memory.
|
||||
/// MIGRATION-ONLY as of LocalDb Phase 1. <see cref="OperationTrackingStore"/> no longer
|
||||
/// reads this: the tracking table lives in the consolidated site database at
|
||||
/// <c>LocalDb:Path</c>. The only remaining consumer is
|
||||
/// <c>SiteLocalDbLegacyMigrator</c>, which uses it to locate a pre-Phase-1
|
||||
/// <c>site-tracking.db</c> and copy it in once. Safe to delete from a config once that
|
||||
/// node's migration has run.
|
||||
/// </summary>
|
||||
public string ConnectionString { get; set; } = "Data Source=site-tracking.db";
|
||||
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.LocalDb.Replication;
|
||||
using ZB.MOM.WW.ScadaBridge.Host;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// LocalDb Phase 1 (Task 8) — the passive sync endpoint's inbound gate.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The replication library's <c>LocalDbSyncService</c> verifies nothing; inbound auth is
|
||||
/// explicitly the host's job. Without this interceptor, anything able to reach a site
|
||||
/// node's gRPC port could stream arbitrary rows straight into the consolidated site
|
||||
/// database — including into <c>OperationTracking</c>, which central reconciles from.
|
||||
/// </remarks>
|
||||
public class LocalDbSyncAuthInterceptorTests
|
||||
{
|
||||
private const string SyncMethod = "/localdb_sync.v1.LocalDbSync/Sync";
|
||||
private const string SiteStreamMethod = "/scadabridge.SiteStream/Connect";
|
||||
|
||||
private static LocalDbSyncAuthInterceptor CreateInterceptor(string? apiKey)
|
||||
=> new(
|
||||
Options.Create(new ReplicationOptions { ApiKey = apiKey }),
|
||||
NullLogger<LocalDbSyncAuthInterceptor>.Instance);
|
||||
|
||||
private static ServerCallContext CreateContext(string method, string? authorizationHeader)
|
||||
{
|
||||
var headers = new Metadata();
|
||||
if (authorizationHeader is not null)
|
||||
headers.Add("authorization", authorizationHeader);
|
||||
|
||||
return new FakeServerCallContext(method, headers);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal <see cref="ServerCallContext"/> carrying just a method name and request
|
||||
/// headers — the only two things the interceptor reads.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Hand-rolled rather than using <c>Grpc.Core.Testing.TestServerCallContext</c>: that
|
||||
/// type ships in the retired native <c>Grpc.Core</c> package and does not exist on the
|
||||
/// grpc-dotnet stack this solution runs on. Pulling in the dead package to get one
|
||||
/// test helper would be a worse trade than these few overrides.
|
||||
/// </remarks>
|
||||
private sealed class FakeServerCallContext(string method, Metadata requestHeaders)
|
||||
: ServerCallContext
|
||||
{
|
||||
protected override string MethodCore => method;
|
||||
protected override string HostCore => "localhost";
|
||||
protected override string PeerCore => "ipv4:127.0.0.1:12345";
|
||||
protected override DateTime DeadlineCore => DateTime.UtcNow.AddMinutes(1);
|
||||
protected override Metadata RequestHeadersCore => requestHeaders;
|
||||
protected override CancellationToken CancellationTokenCore => CancellationToken.None;
|
||||
protected override Metadata ResponseTrailersCore { get; } = [];
|
||||
protected override Status StatusCore { get; set; }
|
||||
protected override WriteOptions? WriteOptionsCore { get; set; }
|
||||
protected override AuthContext AuthContextCore { get; } =
|
||||
new(null, new Dictionary<string, List<AuthProperty>>());
|
||||
|
||||
protected override ContextPropagationToken CreatePropagationTokenCore(
|
||||
ContextPropagationOptions? options)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Invokes the interceptor's unary path with a trivial continuation.</summary>
|
||||
private static Task<string> Invoke(
|
||||
LocalDbSyncAuthInterceptor interceptor, ServerCallContext context)
|
||||
=> interceptor.UnaryServerHandler<string, string>(
|
||||
"request", context, (_, _) => Task.FromResult("ok"));
|
||||
|
||||
[Fact]
|
||||
public async Task NonSyncMethod_PassesThrough_EvenWithNoKeyConfigured()
|
||||
{
|
||||
// The interceptor is registered on the shared site AddGrpc pipeline, so it sees
|
||||
// every call. It must be scoped strictly to the sync service — gating SiteStream
|
||||
// would break site↔central communication outright.
|
||||
var interceptor = CreateInterceptor(apiKey: null);
|
||||
var context = CreateContext(SiteStreamMethod, authorizationHeader: null);
|
||||
|
||||
Assert.Equal("ok", await Invoke(interceptor, context));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SyncMethod_WithNoKeyConfigured_IsDenied_EvenWithABearerToken()
|
||||
{
|
||||
// Fail-closed. "No key configured" is the DEFAULT every site node ships with, so
|
||||
// treating it as "no auth required" would silently expose the endpoint on exactly
|
||||
// the configuration that is most common. Presenting a token must not help.
|
||||
var interceptor = CreateInterceptor(apiKey: null);
|
||||
var context = CreateContext(SyncMethod, "Bearer anything-at-all");
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(() => Invoke(interceptor, context));
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SyncMethod_WithNoBearerToken_IsDenied()
|
||||
{
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(SyncMethod, authorizationHeader: null);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(() => Invoke(interceptor, context));
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SyncMethod_WithWrongBearerToken_IsDenied()
|
||||
{
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(SyncMethod, "Bearer the-wrong-key");
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(() => Invoke(interceptor, context));
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SyncMethod_WithCorrectBearerToken_PassesThrough()
|
||||
{
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(SyncMethod, "Bearer the-shared-key");
|
||||
|
||||
Assert.Equal("ok", await Invoke(interceptor, context));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SyncMethod_WithCorrectKey_ButNoBearerScheme_IsDenied()
|
||||
{
|
||||
// A raw key with no "Bearer " prefix is not what SyncBackgroundService sends, and
|
||||
// accepting it would widen the accepted credential shape for no reason.
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(SyncMethod, "the-shared-key");
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(() => Invoke(interceptor, context));
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SyncMethod_TokenComparison_IsNotAPrefixMatch()
|
||||
{
|
||||
// A prefix/StartsWith comparison would accept a truncated key and make the secret
|
||||
// recoverable one character at a time.
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(SyncMethod, "Bearer the-shared-ke");
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(() => Invoke(interceptor, context));
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DuplexStreaming_IsGated_BecauseThatIsHowSyncActuallyRuns()
|
||||
{
|
||||
// The sync RPC is a bidirectional stream. Gating only the unary path would leave
|
||||
// the real endpoint wide open while every unary test still passed.
|
||||
var interceptor = CreateInterceptor("the-shared-key");
|
||||
var context = CreateContext(SyncMethod, "Bearer the-wrong-key");
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(() =>
|
||||
interceptor.DuplexStreamingServerHandler<string, string>(
|
||||
requestStream: null!,
|
||||
responseStream: null!,
|
||||
context,
|
||||
(_, _, _) => Task.CompletedTask));
|
||||
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.ScadaBridge.Host;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// LocalDb Phase 1 (Task 6) — the one-time copy of the pre-Phase-1 site databases into
|
||||
/// the consolidated one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is a data migration that runs during host startup, so its failure modes are
|
||||
/// expensive: a duplicate-on-rerun bug silently doubles a site's event history, and a
|
||||
/// half-committed copy leaves an operator with no clean state to recover to. Every bullet
|
||||
/// of the plan's semantics gets its own test.
|
||||
/// </remarks>
|
||||
public class SiteLocalDbLegacyMigratorTests : IDisposable
|
||||
{
|
||||
private readonly string _root;
|
||||
private readonly List<ServiceProvider> _providers = [];
|
||||
|
||||
public SiteLocalDbLegacyMigratorTests()
|
||||
{
|
||||
_root = Path.Combine(Path.GetTempPath(), $"localdb-migrator-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_root);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var provider in _providers)
|
||||
{
|
||||
try { provider.Dispose(); } catch { /* best effort */ }
|
||||
}
|
||||
|
||||
try { Directory.Delete(_root, recursive: true); } catch { /* best effort */ }
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
private string Path_(string name) => System.IO.Path.Combine(_root, name);
|
||||
|
||||
/// <summary>A consolidated database with both tables created and registered, as the host has it.</summary>
|
||||
private ILocalDb CreateConsolidated(IConfiguration config)
|
||||
{
|
||||
var provider = new ServiceCollection()
|
||||
.AddZbLocalDb(config, db =>
|
||||
{
|
||||
using var connection = db.CreateConnection();
|
||||
ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking.OperationTrackingSchema.Apply(connection);
|
||||
ZB.MOM.WW.ScadaBridge.SiteEventLogging.SiteEventLogSchema.Apply(connection);
|
||||
db.RegisterReplicated("OperationTracking");
|
||||
db.RegisterReplicated("site_events");
|
||||
})
|
||||
.BuildServiceProvider();
|
||||
_providers.Add(provider);
|
||||
|
||||
return provider.GetRequiredService<ILocalDb>();
|
||||
}
|
||||
|
||||
private IConfiguration Config(
|
||||
string? trackingPath = null, string? eventsPath = null, string nodeName = "node-a")
|
||||
{
|
||||
var values = new Dictionary<string, string?>
|
||||
{
|
||||
["LocalDb:Path"] = Path_("consolidated.db"),
|
||||
["ScadaBridge:Node:NodeName"] = nodeName,
|
||||
};
|
||||
|
||||
if (trackingPath is not null)
|
||||
values["ScadaBridge:OperationTracking:ConnectionString"] = $"Data Source={trackingPath}";
|
||||
if (eventsPath is not null)
|
||||
values["ScadaBridge:SiteEventLog:DatabasePath"] = eventsPath;
|
||||
|
||||
return new ConfigurationBuilder().AddInMemoryCollection(values).Build();
|
||||
}
|
||||
|
||||
private static void SeedLegacyTracking(string path, params string[] ids)
|
||||
{
|
||||
using var connection = new SqliteConnection($"Data Source={path}");
|
||||
connection.Open();
|
||||
ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking.OperationTrackingSchema.Apply(connection);
|
||||
|
||||
foreach (var id in ids)
|
||||
{
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
INSERT INTO OperationTracking (
|
||||
TrackedOperationId, Kind, TargetSummary, Status, RetryCount,
|
||||
CreatedAtUtc, UpdatedAtUtc)
|
||||
VALUES ($id, 'ApiCallCached', 'ERP.GetOrder', 'Submitted', 0, $now, $now);
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$id", id);
|
||||
cmd.Parameters.AddWithValue("$now", DateTime.UtcNow.ToString("o"));
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Seeds the LEGACY event schema: an autoincrement integer id, not a GUID.</summary>
|
||||
private static void SeedLegacyEvents(string path, int count)
|
||||
{
|
||||
using var connection = new SqliteConnection($"Data Source={path}");
|
||||
connection.Open();
|
||||
|
||||
using (var ddl = connection.CreateCommand())
|
||||
{
|
||||
ddl.CommandText = """
|
||||
CREATE TABLE IF NOT EXISTS site_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL, event_type TEXT NOT NULL, severity TEXT NOT NULL,
|
||||
instance_id TEXT, source TEXT NOT NULL, message TEXT NOT NULL, details TEXT
|
||||
);
|
||||
""";
|
||||
ddl.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
INSERT INTO site_events (timestamp, event_type, severity, source, message)
|
||||
VALUES ($ts, 'script', 'Info', 'src', $msg);
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$ts", DateTimeOffset.UtcNow.AddMinutes(-i).ToString("o"));
|
||||
cmd.Parameters.AddWithValue("$msg", $"legacy message {i}");
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
private static long CountRows(ILocalDb db, string table)
|
||||
{
|
||||
using var connection = db.CreateConnection();
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = $"SELECT COUNT(*) FROM {table}";
|
||||
return (long)cmd.ExecuteScalar()!;
|
||||
}
|
||||
|
||||
private static List<string> SelectIds(ILocalDb db, string table, string idColumn)
|
||||
{
|
||||
using var connection = db.CreateConnection();
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = $"SELECT {idColumn} FROM {table} ORDER BY {idColumn}";
|
||||
using var reader = cmd.ExecuteReader();
|
||||
|
||||
var ids = new List<string>();
|
||||
while (reader.Read()) ids.Add(reader.GetString(0));
|
||||
return ids;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MissingLegacyFiles_AreANoOp()
|
||||
{
|
||||
// The expected case on the docker rig: neither legacy key is set anywhere, and
|
||||
// the CWD-relative defaults point outside the data volume, so there is usually
|
||||
// nothing there at all. That must be a clean no-op, not a startup failure.
|
||||
var config = Config(Path_("absent-tracking.db"), Path_("absent-events.db"));
|
||||
var db = CreateConsolidated(config);
|
||||
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
|
||||
Assert.Equal(0, CountRows(db, "OperationTracking"));
|
||||
Assert.Equal(0, CountRows(db, "site_events"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LegacyTrackingRows_AreCopiedAndTheFileIsRenamed()
|
||||
{
|
||||
var trackingPath = Path_("site-tracking.db");
|
||||
SeedLegacyTracking(trackingPath, "op-1", "op-2", "op-3");
|
||||
var config = Config(trackingPath, Path_("absent-events.db"));
|
||||
var db = CreateConsolidated(config);
|
||||
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
|
||||
Assert.Equal(3, CountRows(db, "OperationTracking"));
|
||||
Assert.Equal(["op-1", "op-2", "op-3"], SelectIds(db, "OperationTracking", "TrackedOperationId"));
|
||||
|
||||
// Renamed, not deleted — the operator can still inspect the original.
|
||||
Assert.False(File.Exists(trackingPath));
|
||||
Assert.True(File.Exists(trackingPath + ".migrated"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LegacyEvents_GetDeterministicIds_NotFreshGuids()
|
||||
{
|
||||
// The whole reason a rerun cannot duplicate. Fresh GUIDs would make
|
||||
// INSERT OR IGNORE useless and double the history on every retry.
|
||||
var eventsPath = Path_("site_events.db");
|
||||
SeedLegacyEvents(eventsPath, count: 3);
|
||||
var config = Config(Path_("absent-tracking.db"), eventsPath, nodeName: "node-b");
|
||||
var db = CreateConsolidated(config);
|
||||
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
|
||||
Assert.Equal(
|
||||
["mig-node-b-1", "mig-node-b-2", "mig-node-b-3"],
|
||||
SelectIds(db, "site_events", "id"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SecondBoot_IsANoOp_BecauseTheFileIsAlreadyMarkedMigrated()
|
||||
{
|
||||
var trackingPath = Path_("site-tracking.db");
|
||||
var eventsPath = Path_("site_events.db");
|
||||
SeedLegacyTracking(trackingPath, "op-1", "op-2");
|
||||
SeedLegacyEvents(eventsPath, count: 2);
|
||||
var config = Config(trackingPath, eventsPath);
|
||||
var db = CreateConsolidated(config);
|
||||
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
|
||||
Assert.Equal(2, CountRows(db, "OperationTracking"));
|
||||
Assert.Equal(2, CountRows(db, "site_events"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RerunAgainstAnUnrenamedFile_DoesNotDuplicate()
|
||||
{
|
||||
// Simulates the crash window: the copy committed but the rename did not. On the
|
||||
// next boot the migrator sees the legacy file again and re-copies it. Both tables
|
||||
// must absorb that with no duplicates — tracking via its natural TEXT key, events
|
||||
// via the deterministic mig- id.
|
||||
var trackingPath = Path_("site-tracking.db");
|
||||
var eventsPath = Path_("site_events.db");
|
||||
SeedLegacyTracking(trackingPath, "op-1", "op-2");
|
||||
SeedLegacyEvents(eventsPath, count: 2);
|
||||
var config = Config(trackingPath, eventsPath);
|
||||
var db = CreateConsolidated(config);
|
||||
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
|
||||
// Undo the rename, exactly as a crash between commit and rename would leave it.
|
||||
File.Move(trackingPath + ".migrated", trackingPath);
|
||||
File.Move(eventsPath + ".migrated", eventsPath);
|
||||
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
|
||||
Assert.Equal(2, CountRows(db, "OperationTracking"));
|
||||
Assert.Equal(2, CountRows(db, "site_events"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MigratedRows_EnterTheOplog_SoTheyActuallyReplicate()
|
||||
{
|
||||
// The reason the migrator runs AFTER RegisterReplicated. Capture is trigger-based:
|
||||
// migrate before registration and the rows are invisible to the peer forever, with
|
||||
// no error anywhere. Asserting on the oplog is the only way to catch that ordering
|
||||
// being reversed — every other test here would still pass.
|
||||
var trackingPath = Path_("site-tracking.db");
|
||||
SeedLegacyTracking(trackingPath, "op-1", "op-2");
|
||||
var config = Config(trackingPath, Path_("absent-events.db"));
|
||||
var db = CreateConsolidated(config);
|
||||
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
|
||||
using var connection = db.CreateConnection();
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText =
|
||||
"SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'OperationTracking'";
|
||||
Assert.Equal(2L, (long)cmd.ExecuteScalar()!);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LegacyFileWithoutTheExpectedTable_IsTreatedAsEmpty_NotAFailure()
|
||||
{
|
||||
// A file predating the table (or an unrecognised shape) must not fail host boot.
|
||||
var trackingPath = Path_("site-tracking.db");
|
||||
using (var connection = new SqliteConnection($"Data Source={trackingPath}"))
|
||||
{
|
||||
connection.Open();
|
||||
using var ddl = connection.CreateCommand();
|
||||
ddl.CommandText = "CREATE TABLE something_else (x INTEGER);";
|
||||
ddl.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
var config = Config(trackingPath, Path_("absent-events.db"));
|
||||
var db = CreateConsolidated(config);
|
||||
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
|
||||
Assert.Equal(0, CountRows(db, "OperationTracking"));
|
||||
Assert.True(File.Exists(trackingPath + ".migrated"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InMemoryLegacyConnectionStrings_AreSkipped()
|
||||
{
|
||||
// Test/dev configurations point the legacy stores at in-memory databases. There is
|
||||
// nothing durable to migrate, and Path.GetFullPath on ":memory:" would be nonsense.
|
||||
var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["LocalDb:Path"] = Path_("consolidated.db"),
|
||||
["ScadaBridge:Node:NodeName"] = "node-a",
|
||||
["ScadaBridge:OperationTracking:ConnectionString"] =
|
||||
"Data Source=file:legacy?mode=memory&cache=shared",
|
||||
["ScadaBridge:SiteEventLog:DatabasePath"] = ":memory:",
|
||||
}).Build();
|
||||
|
||||
Assert.Equal(string.Empty, SiteLocalDbLegacyMigrator.ResolveTrackingPath(config));
|
||||
Assert.Equal(string.Empty, SiteLocalDbLegacyMigrator.ResolveEventLogPath(config));
|
||||
|
||||
var db = CreateConsolidated(config);
|
||||
SiteLocalDbLegacyMigrator.Migrate(db, config);
|
||||
|
||||
Assert.Equal(0, CountRows(db, "site_events"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnsetLegacyKeys_FallBackToTheCwdRelativeCodeDefaults()
|
||||
{
|
||||
// Neither key is set in any docker appsettings, so the OLD code used these
|
||||
// CWD-relative defaults. Resolving them anywhere else (e.g. against the data
|
||||
// volume) would look for the legacy data in a directory it was never written to.
|
||||
var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["LocalDb:Path"] = Path_("consolidated.db"),
|
||||
}).Build();
|
||||
|
||||
Assert.Equal(
|
||||
System.IO.Path.GetFullPath("site-tracking.db"),
|
||||
SiteLocalDbLegacyMigrator.ResolveTrackingPath(config));
|
||||
Assert.Equal(
|
||||
System.IO.Path.GetFullPath("site_events.db"),
|
||||
SiteLocalDbLegacyMigrator.ResolveEventLogPath(config));
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,13 @@
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.Telemetry;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.LocalDb.Replication;
|
||||
using ZB.MOM.WW.ScadaBridge.Host;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
|
||||
@@ -120,6 +126,95 @@ public class SiteLocalDbWiringTests : IDisposable
|
||||
Assert.Same(first, second);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Site_Replication_IsRegistered_AndInertWithNoPeer()
|
||||
{
|
||||
// Task 7's default-OFF pin. This fixture configures NO
|
||||
// LocalDb:Replication:PeerAddress, which is the shipping default, so the
|
||||
// engine must resolve cleanly and sit idle rather than throw, retry, or
|
||||
// report a connection. "Default-off" here means inert, NOT unregistered —
|
||||
// the services are always in the graph.
|
||||
var status = _host.Services.GetService<ISyncStatus>();
|
||||
|
||||
Assert.NotNull(status);
|
||||
Assert.False(status!.Connected);
|
||||
Assert.Null(status.PeerNodeId);
|
||||
Assert.Null(status.LastSyncUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Site_Replication_OplogBacklog_IsZero_NotNull_OnAHealthyIdleNode()
|
||||
{
|
||||
// OplogBacklog is deliberately nullable: null means "unknown" (no provider
|
||||
// wired, or the poll failed) and must never be reported as a healthy 0. On a
|
||||
// node whose local database is present and readable, a real 0 proves the
|
||||
// backlog provider is actually wired to the oplog — a null here would mean
|
||||
// Task 9's health signal is about to publish "unknown" forever.
|
||||
var status = _host.Services.GetRequiredService<ISyncStatus>();
|
||||
|
||||
Assert.Equal(0L, status.OplogBacklog);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Site_Replication_HealthBridge_IsRegisteredAsAHostedService()
|
||||
{
|
||||
// Task 9. Registered via a factory lambda, so ImplementationType is null and the
|
||||
// only honest assertion is on the resolved instance.
|
||||
var hosted = _host.Services.GetServices<IHostedService>();
|
||||
|
||||
Assert.Contains(hosted, h => h is LocalDbReplicationStatusReporter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Site_Replication_HealthBridge_PublishesStatusOntoTheHealthReport()
|
||||
{
|
||||
// End-to-end through the REAL collector: probe once, then collect. This is what
|
||||
// catches a bridge that is registered but wired to the wrong provider — the
|
||||
// registration test above would pass either way.
|
||||
var reporter = _host.Services.GetServices<IHostedService>()
|
||||
.OfType<LocalDbReplicationStatusReporter>()
|
||||
.Single();
|
||||
var collector = _host.Services.GetRequiredService<ISiteHealthCollector>();
|
||||
|
||||
reporter.Probe();
|
||||
var report = collector.CollectReport("TestSite");
|
||||
|
||||
// No peer configured: not connected, and a REAL 0 backlog rather than null.
|
||||
Assert.False(report.LocalDbReplicationConnected);
|
||||
Assert.Equal(0L, report.LocalDbOplogBacklog);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Site_HealthReport_LocalDbFields_AreNull_BeforeTheReporterHasRun()
|
||||
{
|
||||
// Null means "no data yet", and must be distinguishable from a real
|
||||
// "disconnected, zero backlog". A collector that defaulted these to false/0 would
|
||||
// report an unwired node as a healthy connected one.
|
||||
var collector = new SiteHealthCollector();
|
||||
|
||||
var report = collector.CollectReport("TestSite");
|
||||
|
||||
Assert.Null(report.LocalDbReplicationConnected);
|
||||
Assert.Null(report.LocalDbOplogBacklog);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Site_Telemetry_Allowlists_TheLocalDbReplicationMeter()
|
||||
{
|
||||
// ZbTelemetryOptions.Meters is an ALLOWLIST: an unlisted Meter's instruments are
|
||||
// never observed, with no error and no missing-metric warning anywhere. The
|
||||
// replication meter was silently absent from the rig's /metrics scrape until the
|
||||
// live gate looked for it, so pin the allowlist rather than trusting the next
|
||||
// reader to remember.
|
||||
//
|
||||
// This asserts on the allowlist constant, not on OTel's observed-meter set:
|
||||
// AddZbTelemetry applies its options to a local instance and never registers
|
||||
// IOptions, so there is nothing resolvable to interrogate. The end-to-end proof
|
||||
// that localdb_* actually reaches /metrics is the Task 12 live gate; this test
|
||||
// exists to stop the entry being dropped again.
|
||||
Assert.Contains(LocalDbMetrics.MeterName, SiteServiceRegistration.ObservedMeters);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Site_LocalDb_CreatesTheConfiguredFile()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
using System.Net;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Hosting.Server;
|
||||
using Microsoft.AspNetCore.Hosting.Server.Features;
|
||||
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.LocalDb.Replication;
|
||||
using ZB.MOM.WW.ScadaBridge.Host;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Serializes the site-pair convergence tests against each other: each one stands up a real
|
||||
/// Kestrel listener plus two SQLite files, and running them concurrently under CI
|
||||
/// contention is a flakiness risk.
|
||||
/// </summary>
|
||||
[CollectionDefinition("LocalDbSitePairConvergence")]
|
||||
public sealed class LocalDbSitePairConvergenceCollection;
|
||||
|
||||
/// <summary>
|
||||
/// Two ScadaBridge site nodes replicating the consolidated site database over a REAL
|
||||
/// loopback gRPC transport, through the REAL fail-closed auth interceptor.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is the test that answers the question Phase 1 exists to answer: does a site node
|
||||
/// pair actually stop losing operation-tracking and site-event state? Everything upstream
|
||||
/// of it — schema helpers, DI wiring, the interceptor — can be individually green while the
|
||||
/// pair still fails to converge.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// It uses <see cref="SiteLocalDbSetup.OnReady"/>, not a hand-written schema, so the tables,
|
||||
/// their primary keys, and the registration ORDER under test are the ones the host actually
|
||||
/// runs. A separate schema here would prove only that the test agrees with itself.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Offline: no docker, no external services. Loopback Kestrel with h2c.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
[Collection("LocalDbSitePairConvergence")]
|
||||
public sealed class LocalDbSitePairConvergenceTests : IAsyncLifetime
|
||||
{
|
||||
private const string SharedApiKey = "site-pair-convergence-key";
|
||||
private static readonly TimeSpan ConvergeTimeout = TimeSpan.FromSeconds(30);
|
||||
|
||||
private readonly string _pathA = Path.Combine(Path.GetTempPath(), $"sitepairA-{Guid.NewGuid():N}.db");
|
||||
private readonly string _pathB = Path.Combine(Path.GetTempPath(), $"sitepairB-{Guid.NewGuid():N}.db");
|
||||
|
||||
// The databases are owned by the fixture, in their own providers, and registered into the
|
||||
// hosts as pre-constructed instances. MS.DI does not dispose instances it did not create,
|
||||
// so tearing a host down (the offline-peer scenario) leaves the databases intact and
|
||||
// writable — which is exactly what lets node A accumulate writes while B is down.
|
||||
private ServiceProvider _dbProviderA = null!;
|
||||
private ServiceProvider _dbProviderB = null!;
|
||||
|
||||
private IHost? _serverHost; // node B — passive
|
||||
private IHost? _initiatorHost; // node A — dials the peer
|
||||
|
||||
static LocalDbSitePairConvergenceTests() =>
|
||||
// Grpc.Net.Client dials the loopback server over HTTP/2 cleartext.
|
||||
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
|
||||
|
||||
private ILocalDb A => _dbProviderA.GetRequiredService<ILocalDb>();
|
||||
private ILocalDb B => _dbProviderB.GetRequiredService<ILocalDb>();
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_dbProviderA = BuildDatabaseProvider(_pathA, "node-a");
|
||||
_dbProviderB = BuildDatabaseProvider(_pathB, "node-b");
|
||||
|
||||
// Force construction (and therefore OnReady) before anything replicates.
|
||||
_ = A;
|
||||
_ = B;
|
||||
|
||||
await StartPassiveAsync();
|
||||
await StartInitiatorAsync();
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
await StopHostAsync(_initiatorHost);
|
||||
await StopHostAsync(_serverHost);
|
||||
await _dbProviderA.DisposeAsync();
|
||||
await _dbProviderB.DisposeAsync();
|
||||
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
foreach (var path in new[] { _pathA, _pathB })
|
||||
{
|
||||
foreach (var suffix in new[] { "", "-wal", "-shm" })
|
||||
{
|
||||
try { File.Delete(path + suffix); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- fixture internals ------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// A provider owning one consolidated site database, initialized through the host's own
|
||||
/// <see cref="SiteLocalDbSetup.OnReady"/> — same schema, same registration order.
|
||||
/// </summary>
|
||||
private static ServiceProvider BuildDatabaseProvider(string path, string nodeName)
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["LocalDb:Path"] = path,
|
||||
["ScadaBridge:Node:NodeName"] = nodeName,
|
||||
// Point the legacy migrator at paths that do not exist, so it no-ops rather
|
||||
// than picking up stray files from the test working directory.
|
||||
["ScadaBridge:OperationTracking:ConnectionString"] =
|
||||
$"Data Source={Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db")}",
|
||||
["ScadaBridge:SiteEventLog:DatabasePath"] =
|
||||
Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db"),
|
||||
})
|
||||
.Build();
|
||||
|
||||
return new ServiceCollection()
|
||||
.AddZbLocalDb(config, db => SiteLocalDbSetup.OnReady(db, config))
|
||||
.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private static IConfiguration ReplicationConfig(string? peerAddress)
|
||||
{
|
||||
var values = new Dictionary<string, string?>
|
||||
{
|
||||
// Tight flush + bounded reconnect backoff so convergence is observable well
|
||||
// inside the poll deadline. The 60 s production default would let the doubling
|
||||
// backoff overrun it after a peer outage.
|
||||
["LocalDb:Replication:FlushInterval"] = "00:00:00.050",
|
||||
["LocalDb:Replication:ReconnectBackoffMax"] = "00:00:02",
|
||||
// Both nodes share one key — the interceptor is fail-closed, so a mismatch here
|
||||
// turns every scenario below red (verified by deliberately breaking it).
|
||||
["LocalDb:Replication:ApiKey"] = SharedApiKey,
|
||||
};
|
||||
|
||||
if (peerAddress is not null)
|
||||
values["LocalDb:Replication:PeerAddress"] = peerAddress;
|
||||
|
||||
return new ConfigurationBuilder().AddInMemoryCollection(values).Build();
|
||||
}
|
||||
|
||||
private async Task StartPassiveAsync()
|
||||
{
|
||||
var config = ReplicationConfig(peerAddress: null);
|
||||
|
||||
_serverHost = await new HostBuilder()
|
||||
.ConfigureWebHost(web =>
|
||||
{
|
||||
web.UseKestrel(o =>
|
||||
o.Listen(IPAddress.Loopback, 0, listen => listen.Protocols = HttpProtocols.Http2));
|
||||
web.ConfigureServices(services =>
|
||||
{
|
||||
services.AddLogging();
|
||||
services.AddRouting();
|
||||
// The REAL interceptor, not a stand-in. If it rejected legitimate peer
|
||||
// traffic, every scenario below would fail — which is the point.
|
||||
services.AddGrpc(o => o.Interceptors.Add<LocalDbSyncAuthInterceptor>());
|
||||
services.AddSingleton(B);
|
||||
services.AddZbLocalDbReplication(config);
|
||||
});
|
||||
web.Configure(app =>
|
||||
{
|
||||
app.UseRouting();
|
||||
app.UseEndpoints(e => e.MapZbLocalDbSync());
|
||||
});
|
||||
})
|
||||
.StartAsync();
|
||||
}
|
||||
|
||||
private async Task StartInitiatorAsync()
|
||||
{
|
||||
var config = ReplicationConfig(PassiveAddress());
|
||||
|
||||
_initiatorHost = await new HostBuilder()
|
||||
.ConfigureServices(services =>
|
||||
{
|
||||
services.AddLogging();
|
||||
services.AddSingleton(A);
|
||||
services.AddZbLocalDbReplication(config);
|
||||
})
|
||||
.StartAsync();
|
||||
}
|
||||
|
||||
private string PassiveAddress()
|
||||
=> _serverHost!.Services.GetRequiredService<IServer>()
|
||||
.Features.Get<IServerAddressesFeature>()!.Addresses.Single();
|
||||
|
||||
private static async Task StopHostAsync(IHost? host)
|
||||
{
|
||||
if (host is null) return;
|
||||
try { await host.StopAsync(TimeSpan.FromSeconds(5)); } catch { /* teardown */ }
|
||||
host.Dispose();
|
||||
}
|
||||
|
||||
// ---- data helpers -----------------------------------------------------------------
|
||||
|
||||
private static Task WriteTrackingAsync(ILocalDb db, string id, string status, string target)
|
||||
=> db.ExecuteAsync(
|
||||
"""
|
||||
INSERT INTO OperationTracking (
|
||||
TrackedOperationId, Kind, TargetSummary, Status, RetryCount,
|
||||
CreatedAtUtc, UpdatedAtUtc)
|
||||
VALUES (@id, 'ApiCallCached', @target, @status, 0, @now, @now)
|
||||
ON CONFLICT(TrackedOperationId) DO UPDATE SET
|
||||
Status = excluded.Status,
|
||||
TargetSummary = excluded.TargetSummary,
|
||||
UpdatedAtUtc = excluded.UpdatedAtUtc;
|
||||
""",
|
||||
new { id, status, target, now = DateTime.UtcNow.ToString("o") });
|
||||
|
||||
private static Task WriteEventAsync(ILocalDb db, string id, string message)
|
||||
=> db.ExecuteAsync(
|
||||
"""
|
||||
INSERT INTO site_events (id, timestamp, event_type, severity, source, message)
|
||||
VALUES (@id, @ts, 'script', 'Info', 'convergence-test', @message);
|
||||
""",
|
||||
new { id, ts = DateTimeOffset.UtcNow.ToString("o"), message });
|
||||
|
||||
private static async Task<string?> ReadTrackingStatusAsync(ILocalDb db, string id)
|
||||
{
|
||||
var rows = await db.QueryAsync(
|
||||
"SELECT Status FROM OperationTracking WHERE TrackedOperationId = @id",
|
||||
static r => r.GetString(0), new { id });
|
||||
return rows.Count == 0 ? null : rows[0];
|
||||
}
|
||||
|
||||
private static Task<IReadOnlyList<string>> ReadEventIdsAsync(ILocalDb db)
|
||||
=> db.QueryAsync("SELECT id FROM site_events ORDER BY id", static r => r.GetString(0));
|
||||
|
||||
/// <summary>Polls <paramref name="condition"/> until true or the deadline passes.</summary>
|
||||
private static async Task WaitUntilAsync(Func<Task<bool>> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + ConvergeTimeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (await condition()) return;
|
||||
await Task.Delay(50);
|
||||
}
|
||||
|
||||
Assert.Fail($"Timed out after {ConvergeTimeout.TotalSeconds:0}s waiting for: {because}");
|
||||
}
|
||||
|
||||
// ---- scenarios --------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task TrackingRow_WrittenOnA_BecomesReadableOnB()
|
||||
{
|
||||
await WriteTrackingAsync(A, "op-a-1", "Submitted", "ERP.GetOrder");
|
||||
|
||||
await WaitUntilAsync(
|
||||
async () => await ReadTrackingStatusAsync(B, "op-a-1") == "Submitted",
|
||||
"the tracking row written on node A to appear on node B");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TrackingRow_WrittenOnB_BecomesReadableOnA()
|
||||
{
|
||||
// Replication is bidirectional even though only A dials: proving the passive node's
|
||||
// writes flow back is what makes a failover in EITHER direction safe.
|
||||
await WriteTrackingAsync(B, "op-b-1", "Submitted", "ERP.GetOrder");
|
||||
|
||||
await WaitUntilAsync(
|
||||
async () => await ReadTrackingStatusAsync(A, "op-b-1") == "Submitted",
|
||||
"the tracking row written on node B to appear on node A");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SameOperation_UpdatedOnBothNodes_ConvergesToOneWinner()
|
||||
{
|
||||
// Last-writer-wins on the primary key. The specific winner is not asserted — that is
|
||||
// the HLC's business — but the two nodes MUST agree, and must agree on a value one of
|
||||
// them actually wrote rather than a merge of both.
|
||||
await WriteTrackingAsync(A, "op-conflict", "Submitted", "ERP.GetOrder");
|
||||
await WaitUntilAsync(
|
||||
async () => await ReadTrackingStatusAsync(B, "op-conflict") is not null,
|
||||
"the conflicting row to exist on both nodes before it is updated");
|
||||
|
||||
await WriteTrackingAsync(A, "op-conflict", "Delivered", "ERP.GetOrder");
|
||||
await WriteTrackingAsync(B, "op-conflict", "Parked", "ERP.GetOrder");
|
||||
|
||||
await WaitUntilAsync(
|
||||
async () =>
|
||||
{
|
||||
var a = await ReadTrackingStatusAsync(A, "op-conflict");
|
||||
var b = await ReadTrackingStatusAsync(B, "op-conflict");
|
||||
return a is not null && a == b;
|
||||
},
|
||||
"both nodes to converge on one status for the contended operation");
|
||||
|
||||
var winner = await ReadTrackingStatusAsync(A, "op-conflict");
|
||||
Assert.Contains(winner, new[] { "Delivered", "Parked" });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EventsLoggedOnBothNodes_ConvergeToTheUnion_WithNoIdCollisions()
|
||||
{
|
||||
// The reason site_events moved to GUID ids. Under the old autoincrement scheme both
|
||||
// nodes would independently mint id=1, id=2, ... and last-writer-wins on the primary
|
||||
// key would silently DESTROY one node's events instead of merging them. The union
|
||||
// count is the assertion that catches that.
|
||||
var idsFromA = Enumerable.Range(0, 5).Select(_ => Guid.NewGuid().ToString("N")).ToList();
|
||||
var idsFromB = Enumerable.Range(0, 5).Select(_ => Guid.NewGuid().ToString("N")).ToList();
|
||||
|
||||
foreach (var id in idsFromA) await WriteEventAsync(A, id, "from A");
|
||||
foreach (var id in idsFromB) await WriteEventAsync(B, id, "from B");
|
||||
|
||||
var expected = idsFromA.Concat(idsFromB).OrderBy(x => x, StringComparer.Ordinal).ToList();
|
||||
|
||||
await WaitUntilAsync(
|
||||
async () => (await ReadEventIdsAsync(A)).SequenceEqual(expected)
|
||||
&& (await ReadEventIdsAsync(B)).SequenceEqual(expected),
|
||||
"both nodes to hold the union of all 10 events");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PeerOffline_ThenRejoins_CatchesUpOnEverythingItMissed()
|
||||
{
|
||||
// The failover case that motivates Phase 1: one node is down while the other keeps
|
||||
// working, and nothing written during the outage may be lost.
|
||||
await StopHostAsync(_serverHost);
|
||||
_serverHost = null;
|
||||
|
||||
var idsDuringOutage = Enumerable.Range(0, 5).Select(_ => Guid.NewGuid().ToString("N")).ToList();
|
||||
foreach (var id in idsDuringOutage) await WriteEventAsync(A, id, "written while B was down");
|
||||
await WriteTrackingAsync(A, "op-during-outage", "Delivered", "ERP.GetOrder");
|
||||
|
||||
// Node B's database survived the host teardown (pre-constructed instance), so this is
|
||||
// a genuine rejoin rather than a fresh node. It comes back on a NEW loopback port;
|
||||
// the initiator's channel factory re-reads the peer address on each reconnect.
|
||||
await StartPassiveAsync();
|
||||
await StopHostAsync(_initiatorHost);
|
||||
await StartInitiatorAsync();
|
||||
|
||||
await WaitUntilAsync(
|
||||
async () =>
|
||||
{
|
||||
var events = await ReadEventIdsAsync(B);
|
||||
return idsDuringOutage.All(events.Contains)
|
||||
&& await ReadTrackingStatusAsync(B, "op-during-outage") == "Delivered";
|
||||
},
|
||||
"node B to catch up on everything written while it was offline");
|
||||
}
|
||||
}
|
||||
@@ -11,20 +11,27 @@ namespace ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests;
|
||||
public class EventLogCoverageTests : IDisposable
|
||||
{
|
||||
private readonly string _dbPath;
|
||||
private readonly TestLocalDb _localDb;
|
||||
|
||||
public EventLogCoverageTests()
|
||||
{
|
||||
_dbPath = Path.Combine(Path.GetTempPath(), $"test_coverage_{Guid.NewGuid()}.db");
|
||||
_localDb = TestLocalDb.Create(_dbPath);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (File.Exists(_dbPath)) File.Delete(_dbPath);
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(_dbPath);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
// Each call gets its own connection from the one shared local database, so the
|
||||
// loggers these tests dispose mid-test do not tear down the database itself.
|
||||
private SiteEventLogger NewLogger() => new(
|
||||
Options.Create(new SiteEventLogOptions { DatabasePath = _dbPath }),
|
||||
NullLogger<SiteEventLogger>.Instance);
|
||||
NullLogger<SiteEventLogger>.Instance,
|
||||
_localDb.Db);
|
||||
|
||||
private static EventLogQueryRequest MakeRequest() => new(
|
||||
CorrelationId: "corr-err",
|
||||
|
||||
@@ -6,6 +6,7 @@ namespace ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests;
|
||||
public class EventLogPurgeServiceTests : IDisposable
|
||||
{
|
||||
private readonly SiteEventLogger _eventLogger;
|
||||
private readonly TestLocalDb _localDb;
|
||||
private readonly string _dbPath;
|
||||
private readonly SiteEventLogOptions _options;
|
||||
|
||||
@@ -27,15 +28,19 @@ public class EventLogPurgeServiceTests : IDisposable
|
||||
RetentionDays = 30,
|
||||
MaxStorageMb = 1024
|
||||
};
|
||||
_localDb = TestLocalDb.Create(_dbPath);
|
||||
_eventLogger = new SiteEventLogger(
|
||||
Options.Create(_options),
|
||||
NullLogger<SiteEventLogger>.Instance);
|
||||
NullLogger<SiteEventLogger>.Instance,
|
||||
_localDb.Db);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_eventLogger.Dispose();
|
||||
if (File.Exists(_dbPath)) File.Delete(_dbPath);
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(_dbPath);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
private EventLogPurgeService CreatePurgeService(
|
||||
|
||||
@@ -8,6 +8,7 @@ public class EventLogQueryServiceTests : IDisposable
|
||||
{
|
||||
private readonly SiteEventLogger _eventLogger;
|
||||
private readonly EventLogQueryService _queryService;
|
||||
private readonly TestLocalDb _localDb;
|
||||
private readonly string _dbPath;
|
||||
|
||||
public EventLogQueryServiceTests()
|
||||
@@ -18,7 +19,8 @@ public class EventLogQueryServiceTests : IDisposable
|
||||
DatabasePath = _dbPath,
|
||||
QueryPageSize = 500
|
||||
});
|
||||
_eventLogger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance);
|
||||
_localDb = TestLocalDb.Create(_dbPath);
|
||||
_eventLogger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance, _localDb.Db);
|
||||
_queryService = new EventLogQueryService(
|
||||
_eventLogger,
|
||||
options,
|
||||
@@ -28,7 +30,9 @@ public class EventLogQueryServiceTests : IDisposable
|
||||
public void Dispose()
|
||||
{
|
||||
_eventLogger.Dispose();
|
||||
if (File.Exists(_dbPath)) File.Delete(_dbPath);
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(_dbPath);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
private async Task SeedEvents()
|
||||
@@ -367,7 +371,8 @@ public class EventLogQueryServiceTests : IDisposable
|
||||
QueryPageSize = 500,
|
||||
MaxQueryPageSize = 3,
|
||||
});
|
||||
using var logger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance);
|
||||
using var localDb = TestLocalDb.Create(dbPath);
|
||||
using var logger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance, localDb.Db);
|
||||
var query = new EventLogQueryService(logger, options, NullLogger<EventLogQueryService>.Instance);
|
||||
|
||||
try
|
||||
@@ -391,7 +396,8 @@ public class EventLogQueryServiceTests : IDisposable
|
||||
finally
|
||||
{
|
||||
logger.Dispose();
|
||||
if (File.Exists(dbPath)) File.Delete(dbPath);
|
||||
localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(dbPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,12 +13,14 @@ public class SchemaIndexTests : IDisposable
|
||||
private readonly SiteEventLogger _logger;
|
||||
private readonly SqliteConnection _verifyConnection;
|
||||
private readonly string _dbPath;
|
||||
private readonly TestLocalDb _localDb;
|
||||
|
||||
public SchemaIndexTests()
|
||||
{
|
||||
_dbPath = Path.Combine(Path.GetTempPath(), $"test_index_{Guid.NewGuid()}.db");
|
||||
var options = Options.Create(new SiteEventLogOptions { DatabasePath = _dbPath });
|
||||
_logger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance);
|
||||
_localDb = TestLocalDb.Create(_dbPath);
|
||||
_logger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance, _localDb.Db);
|
||||
|
||||
_verifyConnection = new SqliteConnection($"Data Source={_dbPath}");
|
||||
_verifyConnection.Open();
|
||||
@@ -28,7 +30,8 @@ public class SchemaIndexTests : IDisposable
|
||||
{
|
||||
_verifyConnection.Dispose();
|
||||
_logger.Dispose();
|
||||
if (File.Exists(_dbPath)) File.Delete(_dbPath);
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(_dbPath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests;
|
||||
|
||||
@@ -23,7 +25,7 @@ public class ServiceWiringTests : IDisposable
|
||||
public void Dispose()
|
||||
{
|
||||
_provider?.Dispose();
|
||||
if (File.Exists(_dbPath)) File.Delete(_dbPath);
|
||||
TestLocalDb.DeleteFiles(_dbPath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -32,6 +34,15 @@ public class ServiceWiringTests : IDisposable
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
services.Configure<SiteEventLogOptions>(o => o.DatabasePath = _dbPath);
|
||||
// SiteEventLogger now takes ILocalDb, so the consolidated site database has to
|
||||
// be in the graph for AddSiteEventLogging to resolve at all. In the host this
|
||||
// comes from SiteServiceRegistration's AddZbLocalDb.
|
||||
services.AddZbLocalDb(new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["LocalDb:Path"] = _dbPath,
|
||||
})
|
||||
.Build());
|
||||
services.AddSiteEventLogging();
|
||||
|
||||
_provider = services.BuildServiceProvider();
|
||||
@@ -57,8 +68,9 @@ public class ServiceWiringTests : IDisposable
|
||||
// concrete SiteEventLogger. Constructing them directly must compile and work.
|
||||
var options = Options.Create(new SiteEventLogOptions { DatabasePath = _dbPath });
|
||||
using var loggerFactory = LoggerFactory.Create(_ => { });
|
||||
using var localDb = TestLocalDb.Create(_dbPath);
|
||||
using var recorder = new SiteEventLogger(
|
||||
options, loggerFactory.CreateLogger<SiteEventLogger>());
|
||||
options, loggerFactory.CreateLogger<SiteEventLogger>(), localDb.Db);
|
||||
|
||||
var purge = new EventLogPurgeService(
|
||||
recorder, options, loggerFactory.CreateLogger<EventLogPurgeService>());
|
||||
|
||||
@@ -13,18 +13,21 @@ public class SiteEventLoggerAsyncTests : IDisposable
|
||||
{
|
||||
private readonly SiteEventLogger _logger;
|
||||
private readonly string _dbPath;
|
||||
private readonly TestLocalDb _localDb;
|
||||
|
||||
public SiteEventLoggerAsyncTests()
|
||||
{
|
||||
_dbPath = Path.Combine(Path.GetTempPath(), $"test_async_{Guid.NewGuid()}.db");
|
||||
var options = Options.Create(new SiteEventLogOptions { DatabasePath = _dbPath });
|
||||
_logger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance);
|
||||
_localDb = TestLocalDb.Create(_dbPath);
|
||||
_logger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance, _localDb.Db);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_logger.Dispose();
|
||||
if (File.Exists(_dbPath)) File.Delete(_dbPath);
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(_dbPath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -91,11 +94,10 @@ public class SiteEventLoggerAsyncTests : IDisposable
|
||||
// Health Monitoring can surface a logging outage.
|
||||
using var failingConnection = new SqliteConnection("Data Source=:memory:");
|
||||
failingConnection.Open();
|
||||
var options = Options.Create(new SiteEventLogOptions
|
||||
{
|
||||
DatabasePath = Path.Combine(Path.GetTempPath(), $"test_fail_{Guid.NewGuid()}.db")
|
||||
});
|
||||
var logger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance);
|
||||
var failDbPath = Path.Combine(Path.GetTempPath(), $"test_fail_{Guid.NewGuid()}.db");
|
||||
var options = Options.Create(new SiteEventLogOptions { DatabasePath = failDbPath });
|
||||
using var failLocalDb = TestLocalDb.Create(failDbPath);
|
||||
var logger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance, failLocalDb.Db);
|
||||
|
||||
// Drop the table so every write fails with "no such table".
|
||||
logger.WithConnection(connection =>
|
||||
@@ -112,5 +114,7 @@ public class SiteEventLoggerAsyncTests : IDisposable
|
||||
"FailedWriteCount must increment when an event write fails.");
|
||||
|
||||
logger.Dispose();
|
||||
failLocalDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(failDbPath);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,12 +9,14 @@ public class SiteEventLoggerTests : IDisposable
|
||||
private readonly SiteEventLogger _logger;
|
||||
private readonly SqliteConnection _verifyConnection;
|
||||
private readonly string _dbPath;
|
||||
private readonly TestLocalDb _localDb;
|
||||
|
||||
public SiteEventLoggerTests()
|
||||
{
|
||||
_dbPath = Path.Combine(Path.GetTempPath(), $"test_events_{Guid.NewGuid()}.db");
|
||||
var options = Options.Create(new SiteEventLogOptions { DatabasePath = _dbPath });
|
||||
_logger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance);
|
||||
_localDb = TestLocalDb.Create(_dbPath);
|
||||
_logger = new SiteEventLogger(options, NullLogger<SiteEventLogger>.Instance, _localDb.Db);
|
||||
|
||||
// Separate connection for verification queries
|
||||
_verifyConnection = new SqliteConnection($"Data Source={_dbPath}");
|
||||
@@ -25,7 +27,8 @@ public class SiteEventLoggerTests : IDisposable
|
||||
{
|
||||
_verifyConnection.Dispose();
|
||||
_logger.Dispose();
|
||||
if (File.Exists(_dbPath)) File.Delete(_dbPath);
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(_dbPath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// A real <see cref="ILocalDb"/> over a temp file, for tests that construct a
|
||||
/// <see cref="SiteEventLogger"/> directly.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <see cref="SiteEventLogger"/> takes <see cref="ILocalDb"/> rather than a connection
|
||||
/// string, and there is no in-memory mode — <c>LocalDbOptions.Path</c> is a filesystem
|
||||
/// path. A real one is used rather than a stub on purpose: connections handed out by
|
||||
/// <c>ILocalDb</c> carry the <c>zb_hlc_next()</c> UDF that <c>site_events</c>' capture
|
||||
/// triggers call, so a stub returning a bare <c>SqliteConnection</c> would pass these
|
||||
/// tests while failing closed the moment the table is registered for replication.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// No <c>onReady</c> callback and no <c>RegisterReplicated</c>: the logger's own
|
||||
/// <c>InitializeSchema</c> creates the table, which is what keeps a directly-constructed
|
||||
/// logger self-sufficient. Registration is the host's job
|
||||
/// (<c>SiteLocalDbSetup.OnReady</c>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class TestLocalDb : IDisposable
|
||||
{
|
||||
private readonly ServiceProvider _provider;
|
||||
|
||||
private TestLocalDb(ServiceProvider provider, ILocalDb db)
|
||||
{
|
||||
_provider = provider;
|
||||
Db = db;
|
||||
}
|
||||
|
||||
/// <summary>The live local database.</summary>
|
||||
public ILocalDb Db { get; }
|
||||
|
||||
/// <summary>Opens a local database at <paramref name="databasePath"/>.</summary>
|
||||
public static TestLocalDb Create(string databasePath)
|
||||
{
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["LocalDb:Path"] = databasePath,
|
||||
})
|
||||
.Build();
|
||||
|
||||
var provider = new ServiceCollection()
|
||||
.AddZbLocalDb(configuration)
|
||||
.BuildServiceProvider();
|
||||
|
||||
return new TestLocalDb(provider, provider.GetRequiredService<ILocalDb>());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes <paramref name="databasePath"/> and its WAL sidecars. Call only after the
|
||||
/// owning <see cref="TestLocalDb"/> is disposed — the master connection anchors the WAL.
|
||||
/// </summary>
|
||||
public static void DeleteFiles(string databasePath)
|
||||
{
|
||||
foreach (var suffix in new[] { "", "-wal", "-shm" })
|
||||
{
|
||||
try { File.Delete(databasePath + suffix); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() => _provider.Dispose();
|
||||
}
|
||||
Reference in New Issue
Block a user