From 324ba369d9cae00f086d4f02d5e9866e0354ce5e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 18 Aug 2026 08:43:52 -0400 Subject: [PATCH] =?UTF-8?q?infra(docker)+test(siteeventlog):=20close=20reg?= =?UTF-8?q?ister=20tripwires=2079=20+=2093=20=E2=80=94=20rig=20data=20to?= =?UTF-8?q?=20named=20volumes;=20pin=20site=5Fevents=20schema=20to=20the?= =?UTF-8?q?=20cap-model=20calibration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Row 79 executed its own prescribed remedy without waiting for a second corruption: all ten rig data dirs (8 docker/ nodes + 2 docker-env2/ site nodes) move from virtiofs bind mounts to named docker volumes (scadabridge--data), seeded in place with both rigs stopped and live-verified (site-a localDbReplicationConnected=true / backlog 0 over the migrated files; env2 stable, 0 restarts). The feared inspectability loss is moot — OrbStack exposes volumes at ~/OrbStack/docker/volumes/. The rebuild surfaced latent env2 drift (pre-Secrets-0.6.2 image): env2 now carries the x-secrets-store-env posture + central data volumes. Row 93's silent maintenance trap is now loud: SiteEventsSchemaCalibrationTests pins the site_events columns/indexes the EventLogPurgeService size model is calibrated against, failing with a pointer to the memo §4 recalibration on any schema drift. Rows 92 (accepted by design) and 94 (standing tripwire) annotated accordingly. --- docker-env2/docker-compose.yml | 48 +++++++- docker/README.md | 36 +++--- docker/deploy.sh | 11 +- docker/docker-compose.yml | 96 ++++++++++------ docker/teardown.sh | 4 +- docs/deployment/topology-guide.md | 59 ++++++---- .../2026-07-08-deferred-work-register.md | 8 +- .../SiteEventsSchemaCalibrationTests.cs | 105 ++++++++++++++++++ 8 files changed, 284 insertions(+), 83 deletions(-) create mode 100644 tests/ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests/SiteEventsSchemaCalibrationTests.cs diff --git a/docker-env2/docker-compose.yml b/docker-env2/docker-compose.yml index fc7f85d4..6d0ec16c 100644 --- a/docker-env2/docker-compose.yml +++ b/docker-env2/docker-compose.yml @@ -1,3 +1,13 @@ +# Secrets 0.6.2 requires Secrets:SqlitePath to be absolute AND outside the content root +# (/app) — mirrors docker/docker-compose.yml's x-secrets-store-env anchor, including the +# trap documented there: DELETING the key does not fall back to the per-user default, it +# yields the RELATIVE "secrets.db", which fails both rules and the node refuses to boot. +# (env2 ran a stale pre-0.6.2 image until the 2026-08-18 rebuild, which surfaced this.) +# /data is the node's own data volume mounted a second time outside /app, so the store +# shares the durable volume without living under the content root. +x-secrets-store-env: &secrets-store-env + Secrets__SqlitePath: "/data/scadabridge-secrets.db" + services: central-a: image: scadabridge:latest @@ -7,6 +17,7 @@ services: stop_grace_period: 30s container_name: scadabridge-env2-central-a environment: + <<: *secrets-store-env SCADABRIDGE_CONFIG: Central ASPNETCORE_ENVIRONMENT: Development ASPNETCORE_URLS: "http://+:5000" @@ -25,6 +36,10 @@ services: - "9111:8081" # Akka remoting volumes: - ./central-node-a/appsettings.Central.json:/app/appsettings.Central.json:ro + - central-node-a-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app) for the + # secrets store — see the x-secrets-store-env anchor above. + - central-node-a-data:/data - ./central-node-a/logs:/app/logs networks: - scadabridge-net @@ -38,6 +53,7 @@ services: stop_grace_period: 30s container_name: scadabridge-env2-central-b environment: + <<: *secrets-store-env SCADABRIDGE_CONFIG: Central ASPNETCORE_ENVIRONMENT: Development ASPNETCORE_URLS: "http://+:5000" @@ -56,6 +72,10 @@ services: - "9112:8081" # Akka remoting volumes: - ./central-node-b/appsettings.Central.json:/app/appsettings.Central.json:ro + - central-node-b-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app) for the + # secrets store — see the x-secrets-store-env anchor above. + - central-node-b-data:/data - ./central-node-b/logs:/app/logs networks: - scadabridge-net @@ -69,6 +89,7 @@ services: stop_grace_period: 30s container_name: scadabridge-env2-site-x-a environment: + <<: *secrets-store-env SCADABRIDGE_CONFIG: Site # Register row 63 — mirrors docker/docker-compose.yml. Enables the Development-only # DI scope validation (register row 50) on the site nodes, which previously ran @@ -79,7 +100,10 @@ services: - "9123:8083" # gRPC streaming volumes: - ./site-x-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro - - ./site-x-node-a/data:/app/data + - site-x-node-a-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app) for the + # secrets store — see the x-secrets-store-env anchor above. + - site-x-node-a-data:/data - ./site-x-node-a/logs:/app/logs networks: - scadabridge-net @@ -93,6 +117,7 @@ services: stop_grace_period: 30s container_name: scadabridge-env2-site-x-b environment: + <<: *secrets-store-env SCADABRIDGE_CONFIG: Site # Register row 63 — mirrors docker/docker-compose.yml. Enables the Development-only # DI scope validation (register row 50) on the site nodes, which previously ran @@ -103,7 +128,10 @@ services: - "9124:8083" # gRPC streaming volumes: - ./site-x-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro - - ./site-x-node-b/data:/app/data + - site-x-node-b-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app) for the + # secrets store — see the x-secrets-store-env anchor above. + - site-x-node-b-data:/data - ./site-x-node-b/logs:/app/logs networks: - scadabridge-net @@ -125,3 +153,19 @@ services: networks: scadabridge-net: external: true + +# Named volumes for the nodes' SQLite data dirs — mirrors docker/docker-compose.yml +# (register row 79: bind mounts cross OrbStack's virtiofs, which does not give SQLite +# true fsync durability). Host-side inspection via ~/OrbStack/docker/volumes/. +# The central volumes are NEW with the Secrets 0.6.2 posture — env2 centrals previously +# kept data on the container overlay (lost on recreate), which 0.6.2's store-path guard +# exists to prevent. +volumes: + central-node-a-data: + name: scadabridge-env2-central-node-a-data + central-node-b-data: + name: scadabridge-env2-central-node-b-data + site-x-node-a-data: + name: scadabridge-env2-site-x-node-a-data + site-x-node-b-data: + name: scadabridge-env2-site-x-node-b-data diff --git a/docker/README.md b/docker/README.md index d2831de4..85f8cb9e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -97,30 +97,29 @@ docker/ │ └── logs/ ├── site-a-node-a/ │ ├── appsettings.Site.json # Site-A node A configuration -│ ├── data/ # SQLite databases (gitignored) -│ └── logs/ +│ └── logs/ # Serilog file output (gitignored) ├── site-a-node-b/ │ ├── appsettings.Site.json -│ ├── data/ │ └── logs/ ├── site-b-node-a/ │ ├── appsettings.Site.json # Site-B node A configuration -│ ├── data/ │ └── logs/ ├── site-b-node-b/ │ ├── appsettings.Site.json -│ ├── data/ │ └── logs/ ├── site-c-node-a/ │ ├── appsettings.Site.json # Site-C node A configuration -│ ├── data/ │ └── logs/ └── site-c-node-b/ ├── appsettings.Site.json - ├── data/ └── logs/ ``` +SQLite databases live in **named docker volumes** (`scadabridge--data`, one per node — +register row 79), not in the node directories; inspect them from the host (read-only) at +`~/OrbStack/docker/volumes//`. Any `docker//data/` directory still on disk is a +dormant pre-migration (2026-08-18) backup, no longer mounted. + ## gRPC control-plane keys (dev) The site gRPC service (`SiteStreamService` on 8083 — live subscriptions, audit pull, @@ -315,20 +314,27 @@ dotnet run --project src/ZB.MOM.WW.ScadaBridge.CLI -- \ ### Clear Site Data -Remove SQLite databases to reset site state (deployed configs, S&F buffers): +Remove SQLite databases to reset site state (deployed configs, S&F buffers). Data lives in +named docker volumes (`scadabridge--data`, register row 79 — no longer bind-mounted +under `docker//data`), so empty the volume with a throwaway container while the pair +is stopped: ```bash # Single site -rm -rf docker/site-a-node-a/data docker/site-a-node-b/data -docker restart scadabridge-site-a-a scadabridge-site-a-b +docker compose -f docker/docker-compose.yml stop site-a-a site-a-b +docker run --rm \ + -v scadabridge-site-a-node-a-data:/a -v scadabridge-site-a-node-b-data:/b \ + alpine sh -c 'rm -rf /a/* /b/*' +docker compose -f docker/docker-compose.yml start site-a-a site-a-b -# All sites -rm -rf docker/site-*/data -docker restart scadabridge-site-a-a scadabridge-site-a-b \ - scadabridge-site-b-a scadabridge-site-b-b \ - scadabridge-site-c-a scadabridge-site-c-b +# All sites: repeat with the six scadabridge-site-*-data volumes, or teardown + volume rm +# (see teardown.sh's hint) and redeploy. ``` +To inspect a node's live databases from the host, read them via OrbStack's volume +exposure (read-only inspection — don't write through this path): +`~/OrbStack/docker/volumes/scadabridge-site-a-node-a-data/site-localdb.db`. + ### Rebuild Image From Scratch (no cache) ```bash diff --git a/docker/deploy.sh b/docker/deploy.sh index f390d3ab..cb0f30b9 100755 --- a/docker/deploy.sh +++ b/docker/deploy.sh @@ -3,13 +3,14 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -# The compose file's bind mounts are RELATIVE (./site-a-node-a/data etc.), so running -# this script from a git worktree silently rehomes every node's data directory to the -# worktree's copy — the rig keeps running, but its SQLite stores split from the main -# checkout's and all state written there is lost on the next deploy from main. +# The compose file's remaining bind mounts (appsettings, logs) are RELATIVE, so running +# this script from a git worktree silently rehomes every node's config to the worktree's +# copy — the rig keeps running, but on stale/divergent appsettings. (The data dirs moved +# to named volumes — register row 79 — so rig state itself no longer splits, but a +# worktree deploy is still wrong by default.) if [[ "$SCRIPT_DIR" == *"/.claude/worktrees/"* && "${SCADABRIDGE_ALLOW_WORKTREE_DEPLOY:-}" != "1" ]]; then echo "ERROR: refusing to deploy from a git worktree ($SCRIPT_DIR)." >&2 - echo "Bind-mount data dirs would rehome to the worktree copy and split rig state." >&2 + echo "Bind-mounted appsettings would rehome to the worktree copy." >&2 echo "Deploy from the main checkout, or set SCADABRIDGE_ALLOW_WORKTREE_DEPLOY=1 to override." >&2 exit 1 fi diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ee1d358d..2be53936 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -6,9 +6,9 @@ # The value must be absolute AND outside the content root (/app) — Secrets 0.6.2 validates # both at startup, because a store inside the deployment directory is destroyed by an # ordinary upgrade (this wiped the MxGateway key store on 2026-08-09). /data is the node's -# own bind mount, the SAME host directory as /app/data, so an existing store is found -# unmoved; only the path used to reach it changes. Do NOT point this back at /app/data — -# absolute is not sufficient, and the boot will fail. +# own data volume (named volume since register row 79), the SAME volume as /app/data, so an +# existing store is found unmoved; only the path used to reach it changes. Do NOT point +# this back at /app/data — absolute is not sufficient, and the boot will fail. # # Do NOT delete this key expecting the library's per-user default to take over. Measured on # this rig: with a Secrets section present but no SqlitePath, the effective value is the @@ -129,11 +129,11 @@ services: # central store moved to the shared SQL-Server database (Secrets 0.5.0, # scadaproj#4) the scadabridge-secrets.db here is a pre-0.5.0 residue, but the # volume is still needed (inbound-api-keys.sqlite lives on it). - - ./central-node-a/data:/app/data - # Same host directory, mounted a second time OUTSIDE the content root (/app). - # Secrets 0.6.2 refuses a store path under the content root; this bind mount is + - central-node-a-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app). + # Secrets 0.6.2 refuses a store path under the content root; this mount is # not touched by an image rebuild, but the guard is lexical and cannot know that. - - ./central-node-a/data:/data + - central-node-a-data:/data - ./central-node-a/logs:/app/logs networks: - scadabridge-net @@ -190,11 +190,11 @@ services: # central store moved to the shared SQL-Server database (Secrets 0.5.0, # scadaproj#4) the scadabridge-secrets.db here is a pre-0.5.0 residue, but the # volume is still needed (inbound-api-keys.sqlite lives on it). - - ./central-node-b/data:/app/data - # Same host directory, mounted a second time OUTSIDE the content root (/app). - # Secrets 0.6.2 refuses a store path under the content root; this bind mount is + - central-node-b-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app). + # Secrets 0.6.2 refuses a store path under the content root; this mount is # not touched by an image rebuild, but the guard is lexical and cannot know that. - - ./central-node-b/data:/data + - central-node-b-data:/data - ./central-node-b/logs:/app/logs networks: - scadabridge-net @@ -226,11 +226,11 @@ services: - "9023:8083" # gRPC streaming volumes: - ./site-a-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro - - ./site-a-node-a/data:/app/data - # Same host directory, mounted a second time OUTSIDE the content root (/app). - # Secrets 0.6.2 refuses a store path under the content root; this bind mount is + - site-a-node-a-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app). + # Secrets 0.6.2 refuses a store path under the content root; this mount is # not touched by an image rebuild, but the guard is lexical and cannot know that. - - ./site-a-node-a/data:/data + - site-a-node-a-data:/data - ./site-a-node-a/logs:/app/logs networks: - scadabridge-net @@ -262,11 +262,11 @@ services: - "9024:8083" # gRPC streaming volumes: - ./site-a-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro - - ./site-a-node-b/data:/app/data - # Same host directory, mounted a second time OUTSIDE the content root (/app). - # Secrets 0.6.2 refuses a store path under the content root; this bind mount is + - site-a-node-b-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app). + # Secrets 0.6.2 refuses a store path under the content root; this mount is # not touched by an image rebuild, but the guard is lexical and cannot know that. - - ./site-a-node-b/data:/data + - site-a-node-b-data:/data - ./site-a-node-b/logs:/app/logs networks: - scadabridge-net @@ -296,11 +296,11 @@ services: - "9033:8083" # gRPC streaming volumes: - ./site-b-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro - - ./site-b-node-a/data:/app/data - # Same host directory, mounted a second time OUTSIDE the content root (/app). - # Secrets 0.6.2 refuses a store path under the content root; this bind mount is + - site-b-node-a-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app). + # Secrets 0.6.2 refuses a store path under the content root; this mount is # not touched by an image rebuild, but the guard is lexical and cannot know that. - - ./site-b-node-a/data:/data + - site-b-node-a-data:/data - ./site-b-node-a/logs:/app/logs networks: - scadabridge-net @@ -330,11 +330,11 @@ services: - "9034:8083" # gRPC streaming volumes: - ./site-b-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro - - ./site-b-node-b/data:/app/data - # Same host directory, mounted a second time OUTSIDE the content root (/app). - # Secrets 0.6.2 refuses a store path under the content root; this bind mount is + - site-b-node-b-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app). + # Secrets 0.6.2 refuses a store path under the content root; this mount is # not touched by an image rebuild, but the guard is lexical and cannot know that. - - ./site-b-node-b/data:/data + - site-b-node-b-data:/data - ./site-b-node-b/logs:/app/logs networks: - scadabridge-net @@ -364,11 +364,11 @@ services: - "9043:8083" # gRPC streaming volumes: - ./site-c-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro - - ./site-c-node-a/data:/app/data - # Same host directory, mounted a second time OUTSIDE the content root (/app). - # Secrets 0.6.2 refuses a store path under the content root; this bind mount is + - site-c-node-a-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app). + # Secrets 0.6.2 refuses a store path under the content root; this mount is # not touched by an image rebuild, but the guard is lexical and cannot know that. - - ./site-c-node-a/data:/data + - site-c-node-a-data:/data - ./site-c-node-a/logs:/app/logs networks: - scadabridge-net @@ -398,11 +398,11 @@ services: - "9044:8083" # gRPC streaming volumes: - ./site-c-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro - - ./site-c-node-b/data:/app/data - # Same host directory, mounted a second time OUTSIDE the content root (/app). - # Secrets 0.6.2 refuses a store path under the content root; this bind mount is + - site-c-node-b-data:/app/data + # Same named volume, mounted a second time OUTSIDE the content root (/app). + # Secrets 0.6.2 refuses a store path under the content root; this mount is # not touched by an image rebuild, but the guard is lexical and cannot know that. - - ./site-c-node-b/data:/data + - site-c-node-b-data:/data - ./site-c-node-b/logs:/app/logs networks: - scadabridge-net @@ -424,3 +424,29 @@ services: networks: scadabridge-net: external: true + +# Named volumes for every node's SQLite data dir (register row 79). On macOS/OrbStack a +# bind mount crosses virtiofs, which does not give SQLite true fsync durability — both +# site-a LocalDb files corrupted through that layer (row 69). Named volumes live on +# native ext4 inside the OrbStack VM, taking virtiofs out of the write path entirely. +# Host-side inspection still works read-only via ~/OrbStack/docker/volumes/; +# `docker compose down` preserves them; remove with `docker volume rm `. +# The appsettings/logs bind mounts stay: config is ro, and Serilog's append-only files +# have no fsync-durability requirement worth losing `tail -f` from the repo tree for. +volumes: + central-node-a-data: + name: scadabridge-central-node-a-data + central-node-b-data: + name: scadabridge-central-node-b-data + site-a-node-a-data: + name: scadabridge-site-a-node-a-data + site-a-node-b-data: + name: scadabridge-site-a-node-b-data + site-b-node-a-data: + name: scadabridge-site-b-node-a-data + site-b-node-b-data: + name: scadabridge-site-b-node-b-data + site-c-node-a-data: + name: scadabridge-site-c-node-a-data + site-c-node-b-data: + name: scadabridge-site-c-node-b-data diff --git a/docker/teardown.sh b/docker/teardown.sh index e577670a..2bc51e55 100755 --- a/docker/teardown.sh +++ b/docker/teardown.sh @@ -10,8 +10,8 @@ docker compose -f "$SCRIPT_DIR/docker-compose.yml" down echo "" echo "Teardown complete." -echo "Site data (SQLite DBs) and logs are preserved in node directories." +echo "Node data (SQLite DBs) is preserved in named docker volumes; logs in node directories." echo "" echo "To also remove persistent data:" -echo " rm -rf $SCRIPT_DIR/site-a-node-a/data $SCRIPT_DIR/site-a-node-b/data" +echo " docker volume rm \$(docker volume ls -q --filter name=scadabridge-central-node --filter name=scadabridge-site-)" echo " rm -rf $SCRIPT_DIR/*/logs" diff --git a/docs/deployment/topology-guide.md b/docs/deployment/topology-guide.md index 127d8ea9..a24017c6 100644 --- a/docs/deployment/topology-guide.md +++ b/docs/deployment/topology-guide.md @@ -359,18 +359,25 @@ database from its peer rather than letting it rejoin. ### Docker-rig data files on macOS/OrbStack — a virtiofs corruption + availability hazard +**Status: MITIGATED 2026-08-18 (register row 79) — the rigs' data dirs moved to named docker +volumes**, taking virtiofs out of the SQLite write path entirely. The hazard description and the +proven recovery runbook below are retained because they explain *why* the volumes exist, and +because failure mode 2 (the VM idle-sleep) is unaffected by the move. + **Scope: the local `docker/` and `docker-env2/` development rigs on macOS only.** This is a host-level property of the filesystem the rig's data files sit on, **not a ScadaBridge defect** and not something a production Linux deployment is exposed to. It is documented here because it has already cost two databases and half a session, and because the recovery is non-obvious. -Every ScadaBridge data file on the rig rides a **relative bind mount** out of the repo working tree -— `./site-a-node-a/data:/app/data` and its siblings, which is where `site-localdb.db`, -`auditlog.db` and the secrets store all live. On macOS that path crosses **virtiofs** to reach the -Linux VM, and **virtiofs does not give SQLite true `fsync` durability**. SQLite's crash-safety -contract assumes a successful `fsync` means the bytes are on stable storage; when that is not true, -an abrupt container kill mid-write can leave a torn file. `docker compose up --force-recreate` -— which `deploy.sh` uses — is exactly such an abrupt kill. +Until 2026-08-18, every ScadaBridge data file on the rig rode a **relative bind mount** out of the +repo working tree — `./site-a-node-a/data:/app/data` and its siblings, which is where +`site-localdb.db`, `auditlog.db` and the secrets store all live. On macOS that path crosses +**virtiofs** to reach the Linux VM, and **virtiofs does not give SQLite true `fsync` durability**. +SQLite's crash-safety contract assumes a successful `fsync` means the bytes are on stable storage; +when that is not true, an abrupt container kill mid-write can leave a torn file. +`docker compose up --force-recreate` — which `deploy.sh` uses — is exactly such an abrupt kill. +Today the same paths are **named volumes** (`scadabridge--data`, +`scadabridge-env2--data`) on native ext4 inside the OrbStack VM, where `fsync` is real. Two distinct failure modes have been observed live (both 2026-08-15): @@ -421,13 +428,16 @@ be restored from a backup instead. 1. **Stop BOTH nodes of the site pair, together.** Not one at a time — this is the same stop-and-start-together rule as *Site Pair Upgrades* above, for the same reason. 2. **Quarantine the database and its sidecars — do not delete them.** Rename all three, keeping - them for forensics: + them for forensics. The data dir now lives in the node's named volume, so run the renames + through a throwaway container (don't write through the host-side `~/OrbStack` view): ```sh - # per affected node's data directory - mv site-localdb.db site-localdb.db.corrupt-$(date +%F) - mv site-localdb.db-wal site-localdb.db-wal.corrupt-$(date +%F) - mv site-localdb.db-shm site-localdb.db-shm.corrupt-$(date +%F) + # per affected node — volume name is scadabridge--data + docker run --rm -v scadabridge-site-a-node-a-data:/d alpine sh -c ' + cd /d + mv site-localdb.db site-localdb.db.corrupt-$(date +%F) + mv site-localdb.db-wal site-localdb.db-wal.corrupt-$(date +%F) + mv site-localdb.db-shm site-localdb.db-shm.corrupt-$(date +%F)' ``` The `-wal` and `-shm` files **must** move with the main file. Leaving a stale WAL beside a fresh @@ -443,16 +453,25 @@ On the live recovery, both quarantined files were large (394 MB and 416 MB), the `connected: true` / backlog 0, and a schema change in flight at the time additionally proved the digest handshake green. -#### Mitigation options, with their trade-offs +#### Mitigation applied (2026-08-18): named docker volumes -None of these is applied today; the rig deliberately still runs on bind mounts, and the accepted -position is the runbook above. Recorded so the choice is a choice: +The "accept + runbook" position was retired without waiting for a second corruption. All ten +data dirs (8 `docker/` nodes + 2 `docker-env2/` site nodes) now ride named volumes +(`scadabridge--data` / `scadabridge-env2--data`), declared in each compose file with +explicit `name:`s and seeded from the old bind-mount dirs at migration time. What changed the +original trade-off calculus: the assumed cost — losing host-side inspectability — turned out to +be much smaller than documented, because OrbStack exposes every volume on the Mac at +`~/OrbStack/docker/volumes//`, so `site-localdb.db` can still be opened from the +host for **read-only inspection**. (Never write through that path: it reintroduces exactly the +virtiofs layer the volumes exist to avoid.) The appsettings and logs bind mounts deliberately +stayed: config is read-only, and Serilog's append-only log files have no fsync-durability +requirement worth losing `tail -f` from the repo tree for. -| Option | Gains | Costs | -|--------|-------|-------| -| Move rig data dirs to **named docker volumes** | Files live on native ext4 inside the VM, out of virtiofs entirely — removes the corruption mechanism at the source | Loses the host-side inspectability the debugging sessions lean on constantly: no more opening `site-localdb.db` from the Mac, no `ls` of a node's data dir from the repo tree. Requires a compose change. | -| **Disable OrbStack's idle pause** | Removes failure mode 2 (the mid-operation sleep and the virtiofs `refresh_nodeid` storm) | A host-level setting outside this repo, so it cannot be version-controlled or applied by `deploy.sh` — it silently does not exist on a new machine. Does nothing about failure mode 1. | -| **Accept, and use this runbook** (current) | No changes to compose or host config; full host-side inspectability retained | Corruption recurs on the next unlucky `--force-recreate`, and can sit latent until a later upgrade turns it into a crash loop. | +Residual exposure: **failure mode 2 is availability-only and still real** — the VM can still +idle-sleep mid-operation and hang `docker` until `orb restart docker`; data now survives that +on ext4. Disabling OrbStack's idle pause remains an un-version-controllable host setting and was +left alone. The old `docker//data/` dirs were left in place as a dormant pre-migration +backup; they are no longer mounted and can be deleted once the volumes have proven themselves. ### Central-Site Communication diff --git a/docs/plans/2026-07-08-deferred-work-register.md b/docs/plans/2026-07-08-deferred-work-register.md index 0bd9f48d..7a1c3cdd 100644 --- a/docs/plans/2026-07-08-deferred-work-register.md +++ b/docs/plans/2026-07-08-deferred-work-register.md @@ -71,7 +71,7 @@ All 7 fix-now items landed via PLAN-04/05/06/07/08 (verified in review 08 round | 76 | ~~**CLI README's `audit-config query` JSON sample omits `afterStateJson`/`bundleImportId`** *(Low, docs)*~~ **RESOLVED 2026-08-16 (`4ae57bdb`).** Sample now shows both fields with their nullability semantics, and was grown to two entries mirroring the table-output sample below it row-for-row — which is what makes the "the table drops these two fields" sentence checkable. | Row-57 fix (2026-08-15) | Closed 2026-08-16. | — | | 77 | ~~**Test pattern: reflecting `ActorCell.Actor` right after `ResolveOne` is unordered vs child construction** *(Low, test-infra)*~~ **RESOLVED 2026-08-16 — swept clean, no commit needed.** Full `tests/` sweep: `ActorCell` (1 hit — the already-fixed file's comment), `ActorRefWithCell` (1 — fixed file :135), `.Underlying` (1 — same line), `BindingFlags` near actor/cell (2 — same file), `.Cell` (0), `ResolveOne` (3 files: the fixed one; `SiteAlarmLiveCacheServiceTests:432` resolves-then-`Sys.Stop`s, benign; `ReconcileStartsInstanceTests:363` uses the ref only for `IActorRef` equality, benign), `UnderlyingActor` (6 files — all Akka.TestKit `TestActorRef` via `ActorOfAsTestActorRef`, an ORDERED mechanism, not this pattern). The unordered idiom exists in exactly one place and already carries the bounded-deadline retry. | Row-40 session (2026-08-15) | Closed 2026-08-16. | — | | 78 | ~~**Configs arriving via LocalDb replication start no InstanceActor — row 67's gap through a side door** *(Medium-Low, availability; proven live 2026-08-15)*~~ **RESOLVED 2026-08-16 (`e606b24a`, with row 72 — the DM instance sweep).** A periodic sweep on the Deployment Manager **singleton** (`ScadaBridge:SiteRuntime:InstanceSweepIntervalSeconds`, default 60, validated `>= 0`, `0` disables — default is live everywhere with no rig config change, which is intended): each tick re-reads the durable rows off-thread and, on the actor thread, starts any **enabled** stored config with no running actor. Arrival-path-agnostic by design — covers replication seeding, post-boot snapshot resyncs, and any future path. A LocalDb change hook was considered and rejected on evidence: it covers only the replication path and only the replicated half of the fleet (site-b/c aren't), AND `ILocalDb` 0.3.0 exposes no change-notification API at all, so a hook means a cross-repo upstream capability to close a site-local gap. Warning site event on a real heal names the sweep as the trigger; silent when there is nothing to do. Fail-before (sweep unarmed): 5 of 10 new tests fail including the exact live symptom ("A replication-seeded config for an enabled instance left no running Instance Actor"); 5 negative guards (disabled / in-flight-deploy deferral / matching-revision no-op / silence / interval-0) green in both states. Design note: `docs/plans/2026-08-15-instance-sweep-self-heal-design.md`. **LIVE-PROVEN at close-out (2026-08-16) in the exact original failure sequence:** site-a pair stopped, node-a's LocalDb file quarantined, pair restarted together — node-a's 08:45:55 reconcile reported "0 fetched" (replication had seeded the fresh store; central rightly saw no gap — the precise side door), and at 08:46:50, one sweep interval later, BOTH enabled instances started with the Warning `Instance rig-probe-1 started by deployment-manager sweep self-heal — its config is present in the deployed-config store but no Instance Actor was running until now`; `debug snapshot` live with no extra restart. The probe also exposed the DCL sibling gap — row 85. | This wave's close-out (2026-08-15) | Closed 2026-08-16. | — | -| 79 | **OrbStack virtiofs is a live corruption + availability hazard for the rig's bind-mounted SQLite files** *(Medium, rig infra — host-level, not product)* — **mitigation chosen 2026-08-16: accept + document (`851b528c`).** The full hazard description + the proven quarantine-and-reseed recovery runbook now live in `docs/deployment/topology-guide.md` (new section), including the `orb restart docker` VM-sleep recovery and the named-volumes/idle-pause alternatives with their trade-offs. Row stays OPEN as the hazard tracker: a recurrence should trigger actually moving to named volumes or disabling idle-pause rather than re-running the runbook a third time. Original evidence: the OrbStack VM idle-SLEPT mid-session (`vmgr … msg=sleep`) leaving docker hung ~20 min until `orb restart docker`, with virtiofs `refresh_nodeid: race with new lookup` ERRORs on wake; both site-a LocalDb files (row 69) corrupted through this bind-mount layer — macOS virtiofs does not give SQLite true fsync durability, and abrupt container kills mid-write compound it. | This wave's close-out (2026-08-15); runbook shipped 2026-08-16 | Runbook shipped; hazard accepted for now | **Next corruption occurrence ⇒ stop accepting: move the rig data dirs to named volumes or disable OrbStack idle-pause** | +| 79 | ~~**OrbStack virtiofs is a live corruption + availability hazard for the rig's bind-mounted SQLite files**~~ **RESOLVED 2026-08-18 — the prescribed remedy executed without waiting for a second corruption: all ten rig data dirs (8 `docker/` nodes + 2 `docker-env2/` site nodes) moved to named docker volumes** (`scadabridge--data` / `scadabridge-env2--data`, explicit `name:`s in both compose files), seeded from the old bind dirs with both rigs stopped, then redeployed and live-verified: site-a `localDbReplicationConnected: true` / backlog 0 over the migrated files, deployed instances resubscribed on boot (data continuity proven), env2 stable with 0 restarts. The trade-off that had deferred this — losing host-side inspectability — turned out to be mostly moot: OrbStack exposes every volume on the Mac at `~/OrbStack/docker/volumes//` (read-only inspection; never write through it). The rebuild also surfaced latent env2 drift: env2 was running a pre-Secrets-0.6.2 image, so on the fresh image all four env2 app nodes failed the new store-path validation at boot — fixed by porting the main rig's `x-secrets-store-env` posture (`Secrets__SqlitePath=/data/…` + `/data` second mounts, incl. NEW env2 central data volumes; env2 centrals previously kept data on the ephemeral overlay). Residual accepted: failure mode 2 (VM idle-sleep, `orb restart docker` recovery) is availability-only and unaffected — data now survives it on ext4. Hazard history + recovery runbook retained in `docs/deployment/topology-guide.md` (section updated). Old `docker//data/` dirs left in place as dormant pre-migration backups. | This wave's close-out (2026-08-15); runbook 2026-08-16; migration executed 2026-08-18 | Closed 2026-08-18. | — | | 80 | ~~**Unreplicated sites report NON-NULL `LocalDbReplicationConnected: false` / `LocalDbOplogBacklog: 0` — contradicting the documented nullable semantics** *(Low-Medium, observability correctness)*~~ **RESOLVED 2026-08-16 (`a2f0086f`, with row 73).** Root cause was type-level: `ISiteHealthCollector.SetLocalDbReplicationStatus(bool …)` made null INEXPRESSIBLE for the connected flag, and the Host wired the bridge unconditionally — the replication engine is registered on every site node and idles answering `false`/`0` with no peer. Fix mirrors the `SiteEventLogReplicationCheck` precedent exactly: the Host evaluates `SiteLocalDbSetup.ReplicationIsConfigured` once and passes it as a third provider to `AddLocalDbReplicationHealthBridge` (the SAME captured value now also feeds `SiteEventLogReplicationCheck`, so the two surfaces can't disagree); unconfigured short-circuits to `(null, null)`; the collector widens to `bool?`. The rest of the chain needed NO change — proto already `BoolValue`/`Int64Value` wrappers, `NextBacklogAboveSince` already leaves the anchor unchanged on null, KPI source already conditional: row 33's machinery was correct and had simply never received a null (its null branch was dead code in production until now). `false` now means exactly one thing: configured, peer session down = real fault. Trap that produced the bug, now pinned in tests: configured-ness is `PeerAddress` **OR** `ApiKey` — "no PeerAddress" is NOT "unreplicated" (a passive-half node carries the key alone); the pre-existing Host fixture asserting `false`/`0` was exactly such a node and its assertion is CORRECT — kept with an explaining comment, plus a new genuinely-unreplicated fixture. Fail-before: gate behaviourally reverted → `UnreplicatedSite_HealthReport_ReportsNullForBothLocalDbFields` fails `Expected: null / Actual: False`. | Found at the previous wave's close-out verifying row 33 | Closed 2026-08-16. | — | | 81 | ~~**The site-event-log storage cap cannot be enforced on a replicated node with an undrained oplog** *(Medium-Low, measured 2026-08-16 during the row-34 work)*~~ **RESOLVED 2026-08-17 (`ddb3c32e`) — but NOT via the row's proposed `dbstat` fix, which is impossible on this stack.** Probed BEFORE implementing: the bundled SQLite 3.53.3 has no `ENABLE_DBSTAT_VTAB` (`pragma_module_list` returns 10 modules, none of them `dbstat`/`sqlite_dbpage`); pinned by `SqliteCapabilityTests`, written to FAIL if a future bump enables it (row 94). Switching SQLite builds rejected — the provider is shared with `ZB.MOM.WW.LocalDb` fleet-wide. The cap instead measures `site_events` + its four indexes via a sampled page-layout model: row width sampled 500 rows from each end per tick (`octet_length`), whole-page table cost (linear models under-estimate near-half-page rows by ~40%), clamped by whole-file logical size (a true upper bound). Index pages count against the cap (~34% of footprint — the 1 GB default was sized against a figure that included them). Measured model error −0.8%/+8.4%/+12.8%, biased high on purpose; cross-checks row 34's independent 393 B/row at 397.2. Guard semantics re-based: (1) unsampleable row width ⇒ refuse to trim (checked BEFORE the cap comparison); (2) the old "size did not decrease" guard now runs on the table-scoped estimate, so replication bookkeeping can no longer trip it — it now means exactly one thing: events arriving as fast as the trim removes them. Bytes-per-row held fixed per tick (re-sampling per batch would jitter the estimate through guard 2 — the same self-defeating measurement as the row itself). LocalDb bookkeeping explicitly NOT managed here (`MaxOplogRows`/tombstone retention own it); topology guide gained a volume-sizing subsection stating the honest budget. Fail-before: reverting to whole-file semantics fails 5 new tests and reproduces the exact original signature (rows destroyed, cap still unmet). SiteEventLogging 104/104, Host 520/520. No config keys added or renamed. Decision memo: `docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md`. Model limitations recorded as rows 92/93. | Row-34 measurement harness; §5 of the measurement doc (postscript appended) | Closed 2026-08-17. | — | | 82 | ~~**`AuditLog.Tests` flakes under parallel full-solution runs — `MsSqlMigrationFixture` contention** *(Low, test-infra)*~~ **RESOLVED 2026-08-17 (`da23f2dc`) — three mechanisms, and the row's "per-run database name" prescription was already in place (the fixture has minted `ScadaBridgeAuditMigTest_` per class since it was written).** (1) The headline fingerprint was create+migrate SATURATION: nothing bounded how many fixtures ran `CREATE DATABASE` + the ~100-migration chain at once (measured peak: 45 concurrent test DBs / 19 blocked requests / 98 sessions on one container), pushing DDL past EF's default 30 s command timeout — every observed failure was `..ctor → MigrateAsync → SqlException 258 Execution Timeout Expired`, which xunit reports against every test in the class (hence "29 failures"). Fix: a machine-wide counting semaphore on `sp_getapplock` in `master` (`ScadaBridgeTestMigrate/0..N-1`, 4 slots, `SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS` override) held across create+migrate — the arbiter must live on the shared SERVER because the contention is cross-process; gate connection is deliberately UNPOOLED (a pooled one defers the session-lock release until reuse, stranding a slot); crashed process ⇒ SQL releases the lock; a gate deadline miss proceeds UNGATED so the throttle can never itself be a red. Timeouts raised to 300 s migrate / 120 s create-drop (bounded queue = busy but progressing); once-per-process sweep of orphaned `ScadaBridgeAuditMigTest_*` >2 h; connectivity probe retries 3× (a busy server misread as absent flips every MSSQL test to a green-looking Skip — worse than a red). (2)+(3) Two more flakes behind it: `PartitionPurgeTests` and `NotifyDispatcherAuditTrailTests` LEAKED 100 ms-tick actors (purge / outbox dispatcher) whose only stop was timeout-bounded `TestKit.Dispose` — a leaked dispatcher even delivered the NEXT test's notification through its own exhausted adapter, stranding it `Retrying`. Fixed with `Watch`/`Stop`/`ExpectTerminated` in the creating test. Evidence: 10× concurrent ConfigurationDatabase.Tests went from 10/10 processes failing (17–77 failures each) to 3 consecutive 10/10-green runs; the AuditLog 3×+1× mix from failing to 3 clean runs; standalone AuditLog 383/383, ConfigurationDatabase 395/395 (no runtime regression), SiteCallAudit 66/66 (inherits the fixture); zero orphan DBs or stuck app locks left. Host.Tests/PlaywrightTests hit the PERSISTENT DB on an already-serialized axis — untouched. Systemic residue recorded as rows 95/96. Doc: `docs/test_infra/test_infra_db.md`. | Found independently by two agents in the 2026-08-16 wave | Closed 2026-08-17. | — | @@ -84,9 +84,9 @@ All 7 fix-now items landed via PLAN-04/05/06/07/08 (verified in review 08 round | 89 | ~~**Connection-only artifact deploy does not reach the peer node's DCL**~~ **RESOLVED 2026-08-17 (merge `6cfb6a79`, agent commit `c57e74e6`) — premise corrected TWICE, then fixed.** (1) The standby's DCL is **EMPTY**, not "serving the old endpoint": `DeploymentManagerActor` (role-scoped singleton) is the sole sender of `CreateConnectionCommand` and only populates its own node's `dcl-manager`. The real defect was **create-time and not replication-specific**: `_createdConnections` is per singleton *incarnation*, and `data_connection_definitions` had NO production reader — so any restart/failover rebuilt every connection from the flattened-config snapshots and silently discarded newer connection-only artifact deploys (identical loss on an unreplicated single-node restart). (2) Arbitration decided and shipped: **the definitions store wins wherever it has an opinion; flattened config is authoritative only for names the store never owned** (deploy-only sites unchanged — memo §7.4's objection). A `_connectionDefinitions` overlay refreshes from `ReadStoreSnapshotAsync` at boot AND every sweep tick, applied before instance creation; ping-pong killed by construction (every ensure path resolves the *winner* before hashing, so one config per hash slot). Fail-before: 5/8 new `DclConnectionArbitrationTests` fail with the overlay inert. Decision memo: `docs/plans/2026-08-17-dcl-connection-arbitration-decision.md` (records the accepted residuals: re-adds land at next instance start, not instantly — mirrors the removal's own "bad quality until redeploy" semantic, pinned by test). The discovery that a *live* connection actor is never reconfigured in place on ANY node is a separate pre-existing defect → row 97. **Live rig probe at wave close-out (post-redeploy):** the singleton node logged exactly ONE boot-time `Created DCL connection OPC PLC Simulator` with zero update churn across later sweep ticks (no ping-pong from the overlay), the standby stayed silent (topology as corrected), all three rig-probe-1 tags Good, site-a replication connected / backlog 0. | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — | | 90 | ~~**No removal path for a connection deleted on the peer**~~ **RESOLVED 2026-08-17 (merge `6cfb6a79`, agent commit `c57e74e6`), with row 89.** Both removal paths now share `RemoveConnectionCentrally` + an in-memory tombstone that `EnsureDclConnections` honors — which also closed a pre-existing bug where an artifact removal was **undone by the very next instance start**. The sweep detects a previously-definition-sourced row's disappearance via its own overlay bookkeeping. Accepted residuals, documented in the decision memo: tombstones are not durable across a restart (by decision — matches the long-standing restart-resurrection semantic), and a delete is undetectable on a node that never adopted the row (the stronger "non-empty table ⇒ complete set" inference would stop live connections on a guess; deliberately not taken). | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — | | 91 | ~~**`SiteScriptCompileCacheTests.GetOrAdd_SameCodeAndGlobals_ComputesOnce` is parallel-load fragile**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `26e3a1b9`) — row-74 pattern applied (exemplar `98c94588`), and wider than the named test.** Two habits: a per-test-instance guid key token (no other class can collide with or serve this test's entries, making `factoryCalls`/`Assert.Same` exact and owned) + one-sided bounds on the process-global counters (`>= baseline + n`; safe because all `Clear()` callers share the serialized collection). Four assertions changed, two got STRONGER (`GetOrAdd_SameCode_DifferentGlobals` had a second absolute pair; `Overflow_StaysWithinTheBound` gained an eviction probe over its own keys — a slack bound is not proof a sweep ran; its `ForeignEntrySlack = 512` is a judgement call, the eviction probe is the load-bearing assertion). Sibling absolutes in the same class and in `DeploymentWarmThenGateTests` fixed too. SiteRuntime.Tests run twice at full parallelism, 633/633 both. The underlying process-global `Clear()` cross-talk hazard remains → row 100. | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — | -| 92 | **The event-log cap's size model is sample-based and blind to mid-history width shifts** *(Low, accuracy limitation — documented in memo §3.1)* | Row-81 fix session (2026-08-17) | Row width is sampled 500 rows from each END of `site_events`; a stream whose row width changes sharply mid-history is mis-sized until the trim reaches that region. Self-correcting per tick (each trim exposes new tail rows to the sample), and the model's deliberate +8–13% high bias absorbs modest shifts, but a pathological width profile could over- or under-trim for several ticks. | First field report of the cap trimming visibly wide of its target, or any rework of the size model (see rows 93/94) | -| 93 | **The cap model's page-layout constants are empirical for the CURRENT `site_events` schema** *(Low, maintenance trap)* | Row-81 fix session (2026-08-17) | Fill factors (0.78/0.75), cell overheads (16/8) and especially `IndexedColumnCount = 4` are calibrated to today's table + four indexes. A schema change — new column, new index — silently invalidates them; a fifth index in particular would be silently under-counted. Any `site_events` schema migration must re-run the memo §4 calibration and update the constants together. | Any schema change to `site_events` or its indexes | -| 94 | **`SqliteCapabilityTests` is a deliberate tripwire — do not "fix" it by weakening the assertion** *(Low, standing note)* | Row-81 fix session (2026-08-17) | The test FAILS BY DESIGN if a future SQLite bump ships `dbstat`/`sqlite_dbpage` support. The correct response at that point is to REPLACE the sampled page-layout model with exact `dbstat` measurement (retiring rows 92/93 with it), not to relax the test. The test's own comment says the same. | `SqliteCapabilityTests` fails after a SQLite/provider bump | +| 92 | **The event-log cap's size model is sample-based and blind to mid-history width shifts** *(Low, accuracy limitation — documented in memo §3.1)* | Row-81 fix session (2026-08-17) | Row width is sampled 500 rows from each END of `site_events`; a stream whose row width changes sharply mid-history is mis-sized until the trim reaches that region. Self-correcting per tick (each trim exposes new tail rows to the sample), and the model's deliberate +8–13% high bias absorbs modest shifts, but a pathological width profile could over- or under-trim for several ticks. **Reviewed 2026-08-18: accepted by design** — inherent to any sampled model, not closable by code short of the row-94 `dbstat` replacement; the row-93 schema pin does not (and cannot) cover it. | First field report of the cap trimming visibly wide of its target, or the row-94 `dbstat` replacement (which retires this row with the model) | +| 93 | ~~**The cap model's page-layout constants are empirical for the CURRENT `site_events` schema**~~ **RESOLVED 2026-08-18 — the silent trap is now loud.** `SiteEventsSchemaCalibrationTests` (SiteEventLogging.Tests) pins the exact shape the calibration assumed: the eight-column list the sampler's `octet_length` sum covers, the index set as exactly the four secondary indexes `IndexedColumnCount = 4` charges plus the PK autoindex (absorbed into the measured fill factors), and each index's single covered column. Any `site_events` schema migration — new column, fifth index, composite widening — now fails the suite with a message directing to the memo §4 recalibration and the constants + sampling-SQL update, instead of silently mis-sizing the cap. Suite green 110/110. | Row-81 fix session (2026-08-17); pinned by test 2026-08-18 | Closed 2026-08-18. | — | +| 94 | **`SqliteCapabilityTests` is a deliberate tripwire — do not "fix" it by weakening the assertion** *(Low, standing note)* | Row-81 fix session (2026-08-17) | The test FAILS BY DESIGN if a future SQLite bump ships `dbstat`/`sqlite_dbpage` support. The correct response at that point is to REPLACE the sampled page-layout model with exact `dbstat` measurement (retiring row 92 — and the row-93 schema pin `SiteEventsSchemaCalibrationTests`, added 2026-08-18, goes with the model too), not to relax the test. The test's own comment says the same. | `SqliteCapabilityTests` fails after a SQLite/provider bump | | 95 | ~~**The leaked 100 ms-tick test-actor pattern is systemic — three more classes carry it**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `34a8079d`).** All three classes live in AuditLog.Tests (premise correction: none in SiteCallAudit.Tests). `Watch`/`Stop`/`ExpectTerminated` replicated from `da23f2dc` at every creation site: 11 in `AuditLogPurgeActorTests`, 10 in `SiteAuditReconciliationActorTests`, 2 in `OutageReconciliationTests`. Second premise correction: `OutageReconciliationTests` DID call `Sys.Stop` — but fire-and-forget AND from the last line of the `await using` block, so an in-flight tick could still be writing to MSSQL during read-back and any earlier assertion failure skipped the stop entirely; stops moved before the read-backs, plus the `AuditLogIngestActor` that test 2 was discarding outright (holding a disposed provider) is now stopped too. Known lesser sibling habit left in place: EventStream probes are never unsubscribed (bounded by the per-test ActorSystem — noted, not a row). AuditLog.Tests 383/383. | Row-82 fix session (2026-08-17) | Closed 2026-08-17. | — | | 96 | ~~**xunit parallelism is uncapped almost everywhere**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `d2d4db49`) — policy made and shipped: per-suite caps for the resource-heavy suites, pure unit suites stay uncapped.** The four MSSQL-fixture users (`ConfigurationDatabase.Tests`, `AuditLog.Tests`, `SiteCallAudit.Tests`, `Host.Tests` — verified before capping) get `xunit.runner.json` with `maxParallelThreads: 4`, **deliberately matching row-82's `sp_getapplock` slot default** so the in-process and cross-process limits agree; no collections serialized. `IntegrationTests` (already fully serialized) and `CentralUI.PlaywrightTests` (already `maxParallelThreads: 1`, stricter than needed) left alone. Cap verified live (216% CPU uncapped vs 145% capped on ConfigurationDatabase.Tests); cost is nil — these suites are latency-bound on MSSQL/sockets (14–25% CPU), durations unchanged within noise. Policy documented in `docs/test_infra/test_infra.md` → *Test Parallelism Policy* (two-axis model, per-suite table, default for new projects), cross-linked from `test_infra_db.md`. Found-not-fixed: `StoreAndForward.Tests/xunit.runner.json` is a no-op (pins the default `parallelizeTestCollections: true`) — harmless, tidy on next touch. | Row-82 fix session (2026-08-17) | Closed 2026-08-17. | — | | 97 | ~~**A live DCL connection actor is never reconfigured in place — on ANY node**~~ **RESOLVED 2026-08-17 (merge `9dbda4a2`, agent commit `7c1268d8`) — the memo's §6 sketch shipped, plus two things the sketch missed.** `EnsureDclConnection` now has three outcomes off its one hash slot: unchanged → nothing; new → create; **changed → `ReconfigureConnectionCommand`** (arbitration/tombstones byte-for-byte untouched). The manager forwards to the owning actor (unknown name falls back to create); `DataConnectionActor` handles it in **all three** lifecycle states and never stashes it, via a new shared `SwapAdapter` helper now also used by both failover sites. The sketch's two gaps, found in implementation: **endpoint selection must be re-armed to Primary** (a connection sitting on its old *backup* would otherwise dial the new backup first), and the deliberate link drop must **suppress the unstable-disconnect accounting** (else the swap itself could trip a failover). Also a premise-sharpening: "reconfigure while mid-connect" is the *primary* case, not an edge — a wrong endpoint loops in `Connecting` forever and never reaches `Connected`. Fail-before: 5/7 new `DataConnectionReconfigureTests` + 3 SiteRuntime ensure-path tests fail on the reverted behavior. **Live rig probe (post-redeploy, round trip):** connection 1's config changed (operationTimeoutMs 15000→16000→back) fired exactly TWO in-place reconfigures on the active node — `Reconfiguring DataConnectionActor for OPC PLC Simulator in place` → `Reconnected successfully on Primary endpoint` in 45 ms → `Re-subscribing 2 tags` — with all rig-probe-1 tags Good after each, the standby silent (0 reconfigure lines, topology as designed), and the old dead-end "has a new configuration" Warning never appearing. Rig config restored to original. | Found by the 2026-08-17 rows-89/90 fix session; memo `docs/plans/2026-08-17-dcl-connection-arbitration-decision.md` (§9 postscript appended) | Closed 2026-08-17. | — | diff --git a/tests/ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests/SiteEventsSchemaCalibrationTests.cs b/tests/ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests/SiteEventsSchemaCalibrationTests.cs new file mode 100644 index 00000000..2ac76c7c --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests/SiteEventsSchemaCalibrationTests.cs @@ -0,0 +1,105 @@ +using Microsoft.Data.Sqlite; + +namespace ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests; + +/// +/// Pins the site_events schema to the shape the storage-cap size model was calibrated +/// against (register row 93, docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md §4). +/// +/// +/// hardcodes this schema twice over: +/// the sampling SQL sums octet_length over the exact column list, and +/// IndexedColumnCount = 4 plus the fill-factor/overhead constants are empirical for +/// exactly these four secondary indexes. A schema migration that adds a column or an index +/// invalidates both SILENTLY — the new column's bytes are never sampled, a fifth index is +/// never charged — so this test turns that drift into a loud failure. The fix on failure is +/// NEVER to just update the assertions: re-run the memo §4 calibration, update the +/// constants and sampling SQL together, and only then +/// re-pin the schema here. +/// +public class SiteEventsSchemaCalibrationTests +{ + private const string RecalibrationNotice = + "site_events schema drifted from the size-model calibration. Re-run the calibration in " + + "docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md §4 and update " + + "EventLogPurgeService's constants + sampling SQL together with this pin (register row 93)."; + + private static SqliteConnection OpenWithSchema() + { + var connection = new SqliteConnection("Data Source=:memory:"); + connection.Open(); + SiteEventLogSchema.Apply(connection); + return connection; + } + + private static List ReadStrings(SqliteConnection connection, string sql) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = sql; + var values = new List(); + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + values.Add(reader.GetString(0)); + return values; + } + + [Fact] + public void Columns_MatchTheSampledOctetLengthSum_Exactly() + { + // The sampler's payload_bytes expression sums octet_length over exactly these eight + // columns, in this order. A ninth column would contribute real disk bytes that the + // model never counts, under-sizing every estimate. + using var connection = OpenWithSchema(); + + var columns = ReadStrings(connection, "SELECT name FROM pragma_table_info('site_events')"); + + Assert.True( + columns.SequenceEqual( + ["id", "timestamp", "event_type", "severity", "instance_id", "source", "message", "details"]), + $"{RecalibrationNotice} Actual columns: {string.Join(", ", columns)}"); + } + + [Fact] + public void Indexes_AreExactlyTheFourTheModelCharges_PlusThePkAutoindex() + { + // IndexedColumnCount = 4 charges four secondary index entries per row; the TEXT + // primary key's autoindex is absorbed into the empirically-measured fill factors. + // A fifth secondary index would be silently under-counted. + using var connection = OpenWithSchema(); + + var indexes = ReadStrings( + connection, "SELECT name FROM pragma_index_list('site_events') ORDER BY name"); + + Assert.True( + indexes.SequenceEqual( + [ + "idx_events_instance", + "idx_events_severity", + "idx_events_timestamp", + "idx_events_type", + "sqlite_autoindex_site_events_1", + ]), + $"{RecalibrationNotice} Actual indexes: {string.Join(", ", indexes)}"); + } + + [Theory] + [InlineData("idx_events_timestamp", "timestamp")] + [InlineData("idx_events_type", "event_type")] + [InlineData("idx_events_instance", "instance_id")] + [InlineData("idx_events_severity", "severity")] + public void EachIndex_CoversExactlyTheColumnTheSamplerSums(string indexName, string expectedColumn) + { + // The sampler's indexed_bytes expression sums octet_length(timestamp) + + // octet_length(event_type) + octet_length(severity) + octet_length(instance_id) — + // one column per index. Widening an index to a composite (or re-pointing it) would + // change the real entry width without changing what is sampled. + using var connection = OpenWithSchema(); + + var columns = ReadStrings( + connection, $"SELECT name FROM pragma_index_info('{indexName}')"); + + Assert.True( + columns.SequenceEqual([expectedColumn]), + $"{RecalibrationNotice} Index {indexName} covers: {string.Join(", ", columns)}"); + } +}