infra(docker)+test(siteeventlog): close register tripwires 79 + 93 — rig data to named volumes; pin site_events schema to the cap-model calibration

Row 79 executed its own prescribed remedy without waiting for a second
corruption: all ten rig data dirs (8 docker/ nodes + 2 docker-env2/ site
nodes) move from virtiofs bind mounts to named docker volumes
(scadabridge-<node>-data), seeded in place with both rigs stopped and
live-verified (site-a localDbReplicationConnected=true / backlog 0 over
the migrated files; env2 stable, 0 restarts). The feared inspectability
loss is moot — OrbStack exposes volumes at ~/OrbStack/docker/volumes/.
The rebuild surfaced latent env2 drift (pre-Secrets-0.6.2 image): env2
now carries the x-secrets-store-env posture + central data volumes.

Row 93's silent maintenance trap is now loud: SiteEventsSchemaCalibrationTests
pins the site_events columns/indexes the EventLogPurgeService size model is
calibrated against, failing with a pointer to the memo §4 recalibration on
any schema drift. Rows 92 (accepted by design) and 94 (standing tripwire)
annotated accordingly.
This commit is contained in:
Joseph Doherty
2026-08-18 08:43:52 -04:00
parent 5936573e71
commit 324ba369d9
8 changed files with 284 additions and 83 deletions
+46 -2
View File
@@ -1,3 +1,13 @@
# Secrets 0.6.2 requires Secrets:SqlitePath to be absolute AND outside the content root
# (/app) — mirrors docker/docker-compose.yml's x-secrets-store-env anchor, including the
# trap documented there: DELETING the key does not fall back to the per-user default, it
# yields the RELATIVE "secrets.db", which fails both rules and the node refuses to boot.
# (env2 ran a stale pre-0.6.2 image until the 2026-08-18 rebuild, which surfaced this.)
# /data is the node's own data volume mounted a second time outside /app, so the store
# shares the durable volume without living under the content root.
x-secrets-store-env: &secrets-store-env
Secrets__SqlitePath: "/data/scadabridge-secrets.db"
services: services:
central-a: central-a:
image: scadabridge:latest image: scadabridge:latest
@@ -7,6 +17,7 @@ services:
stop_grace_period: 30s stop_grace_period: 30s
container_name: scadabridge-env2-central-a container_name: scadabridge-env2-central-a
environment: environment:
<<: *secrets-store-env
SCADABRIDGE_CONFIG: Central SCADABRIDGE_CONFIG: Central
ASPNETCORE_ENVIRONMENT: Development ASPNETCORE_ENVIRONMENT: Development
ASPNETCORE_URLS: "http://+:5000" ASPNETCORE_URLS: "http://+:5000"
@@ -25,6 +36,10 @@ services:
- "9111:8081" # Akka remoting - "9111:8081" # Akka remoting
volumes: volumes:
- ./central-node-a/appsettings.Central.json:/app/appsettings.Central.json:ro - ./central-node-a/appsettings.Central.json:/app/appsettings.Central.json:ro
- central-node-a-data:/app/data
# Same named volume, mounted a second time OUTSIDE the content root (/app) for the
# secrets store — see the x-secrets-store-env anchor above.
- central-node-a-data:/data
- ./central-node-a/logs:/app/logs - ./central-node-a/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -38,6 +53,7 @@ services:
stop_grace_period: 30s stop_grace_period: 30s
container_name: scadabridge-env2-central-b container_name: scadabridge-env2-central-b
environment: environment:
<<: *secrets-store-env
SCADABRIDGE_CONFIG: Central SCADABRIDGE_CONFIG: Central
ASPNETCORE_ENVIRONMENT: Development ASPNETCORE_ENVIRONMENT: Development
ASPNETCORE_URLS: "http://+:5000" ASPNETCORE_URLS: "http://+:5000"
@@ -56,6 +72,10 @@ services:
- "9112:8081" # Akka remoting - "9112:8081" # Akka remoting
volumes: volumes:
- ./central-node-b/appsettings.Central.json:/app/appsettings.Central.json:ro - ./central-node-b/appsettings.Central.json:/app/appsettings.Central.json:ro
- central-node-b-data:/app/data
# Same named volume, mounted a second time OUTSIDE the content root (/app) for the
# secrets store — see the x-secrets-store-env anchor above.
- central-node-b-data:/data
- ./central-node-b/logs:/app/logs - ./central-node-b/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -69,6 +89,7 @@ services:
stop_grace_period: 30s stop_grace_period: 30s
container_name: scadabridge-env2-site-x-a container_name: scadabridge-env2-site-x-a
environment: environment:
<<: *secrets-store-env
SCADABRIDGE_CONFIG: Site SCADABRIDGE_CONFIG: Site
# Register row 63 — mirrors docker/docker-compose.yml. Enables the Development-only # Register row 63 — mirrors docker/docker-compose.yml. Enables the Development-only
# DI scope validation (register row 50) on the site nodes, which previously ran # DI scope validation (register row 50) on the site nodes, which previously ran
@@ -79,7 +100,10 @@ services:
- "9123:8083" # gRPC streaming - "9123:8083" # gRPC streaming
volumes: volumes:
- ./site-x-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro - ./site-x-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro
- ./site-x-node-a/data:/app/data - site-x-node-a-data:/app/data
# Same named volume, mounted a second time OUTSIDE the content root (/app) for the
# secrets store — see the x-secrets-store-env anchor above.
- site-x-node-a-data:/data
- ./site-x-node-a/logs:/app/logs - ./site-x-node-a/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -93,6 +117,7 @@ services:
stop_grace_period: 30s stop_grace_period: 30s
container_name: scadabridge-env2-site-x-b container_name: scadabridge-env2-site-x-b
environment: environment:
<<: *secrets-store-env
SCADABRIDGE_CONFIG: Site SCADABRIDGE_CONFIG: Site
# Register row 63 — mirrors docker/docker-compose.yml. Enables the Development-only # Register row 63 — mirrors docker/docker-compose.yml. Enables the Development-only
# DI scope validation (register row 50) on the site nodes, which previously ran # DI scope validation (register row 50) on the site nodes, which previously ran
@@ -103,7 +128,10 @@ services:
- "9124:8083" # gRPC streaming - "9124:8083" # gRPC streaming
volumes: volumes:
- ./site-x-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro - ./site-x-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro
- ./site-x-node-b/data:/app/data - site-x-node-b-data:/app/data
# Same named volume, mounted a second time OUTSIDE the content root (/app) for the
# secrets store — see the x-secrets-store-env anchor above.
- site-x-node-b-data:/data
- ./site-x-node-b/logs:/app/logs - ./site-x-node-b/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -125,3 +153,19 @@ services:
networks: networks:
scadabridge-net: scadabridge-net:
external: true external: true
# Named volumes for the nodes' SQLite data dirs — mirrors docker/docker-compose.yml
# (register row 79: bind mounts cross OrbStack's virtiofs, which does not give SQLite
# true fsync durability). Host-side inspection via ~/OrbStack/docker/volumes/<name>.
# The central volumes are NEW with the Secrets 0.6.2 posture — env2 centrals previously
# kept data on the container overlay (lost on recreate), which 0.6.2's store-path guard
# exists to prevent.
volumes:
central-node-a-data:
name: scadabridge-env2-central-node-a-data
central-node-b-data:
name: scadabridge-env2-central-node-b-data
site-x-node-a-data:
name: scadabridge-env2-site-x-node-a-data
site-x-node-b-data:
name: scadabridge-env2-site-x-node-b-data
+21 -15
View File
@@ -97,30 +97,29 @@ docker/
│ └── logs/ │ └── logs/
├── site-a-node-a/ ├── site-a-node-a/
│ ├── appsettings.Site.json # Site-A node A configuration │ ├── appsettings.Site.json # Site-A node A configuration
│ ├── data/ # SQLite databases (gitignored) │ └── logs/ # Serilog file output (gitignored)
│ └── logs/
├── site-a-node-b/ ├── site-a-node-b/
│ ├── appsettings.Site.json │ ├── appsettings.Site.json
│ ├── data/
│ └── logs/ │ └── logs/
├── site-b-node-a/ ├── site-b-node-a/
│ ├── appsettings.Site.json # Site-B node A configuration │ ├── appsettings.Site.json # Site-B node A configuration
│ ├── data/
│ └── logs/ │ └── logs/
├── site-b-node-b/ ├── site-b-node-b/
│ ├── appsettings.Site.json │ ├── appsettings.Site.json
│ ├── data/
│ └── logs/ │ └── logs/
├── site-c-node-a/ ├── site-c-node-a/
│ ├── appsettings.Site.json # Site-C node A configuration │ ├── appsettings.Site.json # Site-C node A configuration
│ ├── data/
│ └── logs/ │ └── logs/
└── site-c-node-b/ └── site-c-node-b/
├── appsettings.Site.json ├── appsettings.Site.json
├── data/
└── logs/ └── logs/
``` ```
SQLite databases live in **named docker volumes** (`scadabridge-<node>-data`, one per node —
register row 79), not in the node directories; inspect them from the host (read-only) at
`~/OrbStack/docker/volumes/<name>/`. Any `docker/<node>/data/` directory still on disk is a
dormant pre-migration (2026-08-18) backup, no longer mounted.
## gRPC control-plane keys (dev) ## gRPC control-plane keys (dev)
The site gRPC service (`SiteStreamService` on 8083 — live subscriptions, audit pull, The site gRPC service (`SiteStreamService` on 8083 — live subscriptions, audit pull,
@@ -315,20 +314,27 @@ dotnet run --project src/ZB.MOM.WW.ScadaBridge.CLI -- \
### Clear Site Data ### Clear Site Data
Remove SQLite databases to reset site state (deployed configs, S&F buffers): Remove SQLite databases to reset site state (deployed configs, S&F buffers). Data lives in
named docker volumes (`scadabridge-<node>-data`, register row 79 — no longer bind-mounted
under `docker/<node>/data`), so empty the volume with a throwaway container while the pair
is stopped:
```bash ```bash
# Single site # Single site
rm -rf docker/site-a-node-a/data docker/site-a-node-b/data docker compose -f docker/docker-compose.yml stop site-a-a site-a-b
docker restart scadabridge-site-a-a scadabridge-site-a-b docker run --rm \
-v scadabridge-site-a-node-a-data:/a -v scadabridge-site-a-node-b-data:/b \
alpine sh -c 'rm -rf /a/* /b/*'
docker compose -f docker/docker-compose.yml start site-a-a site-a-b
# All sites # All sites: repeat with the six scadabridge-site-*-data volumes, or teardown + volume rm
rm -rf docker/site-*/data # (see teardown.sh's hint) and redeploy.
docker restart scadabridge-site-a-a scadabridge-site-a-b \
scadabridge-site-b-a scadabridge-site-b-b \
scadabridge-site-c-a scadabridge-site-c-b
``` ```
To inspect a node's live databases from the host, read them via OrbStack's volume
exposure (read-only inspection — don't write through this path):
`~/OrbStack/docker/volumes/scadabridge-site-a-node-a-data/site-localdb.db`.
### Rebuild Image From Scratch (no cache) ### Rebuild Image From Scratch (no cache)
```bash ```bash
+6 -5
View File
@@ -3,13 +3,14 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# The compose file's bind mounts are RELATIVE (./site-a-node-a/data etc.), so running # The compose file's remaining bind mounts (appsettings, logs) are RELATIVE, so running
# this script from a git worktree silently rehomes every node's data directory to the # this script from a git worktree silently rehomes every node's config to the worktree's
# worktree's copy — the rig keeps running, but its SQLite stores split from the main # copy — the rig keeps running, but on stale/divergent appsettings. (The data dirs moved
# checkout's and all state written there is lost on the next deploy from main. # to named volumes — register row 79 — so rig state itself no longer splits, but a
# worktree deploy is still wrong by default.)
if [[ "$SCRIPT_DIR" == *"/.claude/worktrees/"* && "${SCADABRIDGE_ALLOW_WORKTREE_DEPLOY:-}" != "1" ]]; then if [[ "$SCRIPT_DIR" == *"/.claude/worktrees/"* && "${SCADABRIDGE_ALLOW_WORKTREE_DEPLOY:-}" != "1" ]]; then
echo "ERROR: refusing to deploy from a git worktree ($SCRIPT_DIR)." >&2 echo "ERROR: refusing to deploy from a git worktree ($SCRIPT_DIR)." >&2
echo "Bind-mount data dirs would rehome to the worktree copy and split rig state." >&2 echo "Bind-mounted appsettings would rehome to the worktree copy." >&2
echo "Deploy from the main checkout, or set SCADABRIDGE_ALLOW_WORKTREE_DEPLOY=1 to override." >&2 echo "Deploy from the main checkout, or set SCADABRIDGE_ALLOW_WORKTREE_DEPLOY=1 to override." >&2
exit 1 exit 1
fi fi
+61 -35
View File
@@ -6,9 +6,9 @@
# The value must be absolute AND outside the content root (/app) — Secrets 0.6.2 validates # The value must be absolute AND outside the content root (/app) — Secrets 0.6.2 validates
# both at startup, because a store inside the deployment directory is destroyed by an # both at startup, because a store inside the deployment directory is destroyed by an
# ordinary upgrade (this wiped the MxGateway key store on 2026-08-09). /data is the node's # ordinary upgrade (this wiped the MxGateway key store on 2026-08-09). /data is the node's
# own bind mount, the SAME host directory as /app/data, so an existing store is found # own data volume (named volume since register row 79), the SAME volume as /app/data, so an
# unmoved; only the path used to reach it changes. Do NOT point this back at /app/data — # existing store is found unmoved; only the path used to reach it changes. Do NOT point
# absolute is not sufficient, and the boot will fail. # this back at /app/data — absolute is not sufficient, and the boot will fail.
# #
# Do NOT delete this key expecting the library's per-user default to take over. Measured on # Do NOT delete this key expecting the library's per-user default to take over. Measured on
# this rig: with a Secrets section present but no SqlitePath, the effective value is the # this rig: with a Secrets section present but no SqlitePath, the effective value is the
@@ -129,11 +129,11 @@ services:
# central store moved to the shared SQL-Server database (Secrets 0.5.0, # central store moved to the shared SQL-Server database (Secrets 0.5.0,
# scadaproj#4) the scadabridge-secrets.db here is a pre-0.5.0 residue, but the # scadaproj#4) the scadabridge-secrets.db here is a pre-0.5.0 residue, but the
# volume is still needed (inbound-api-keys.sqlite lives on it). # volume is still needed (inbound-api-keys.sqlite lives on it).
- ./central-node-a/data:/app/data - central-node-a-data:/app/data
# Same host directory, mounted a second time OUTSIDE the content root (/app). # Same named volume, mounted a second time OUTSIDE the content root (/app).
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is # Secrets 0.6.2 refuses a store path under the content root; this mount is
# not touched by an image rebuild, but the guard is lexical and cannot know that. # not touched by an image rebuild, but the guard is lexical and cannot know that.
- ./central-node-a/data:/data - central-node-a-data:/data
- ./central-node-a/logs:/app/logs - ./central-node-a/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -190,11 +190,11 @@ services:
# central store moved to the shared SQL-Server database (Secrets 0.5.0, # central store moved to the shared SQL-Server database (Secrets 0.5.0,
# scadaproj#4) the scadabridge-secrets.db here is a pre-0.5.0 residue, but the # scadaproj#4) the scadabridge-secrets.db here is a pre-0.5.0 residue, but the
# volume is still needed (inbound-api-keys.sqlite lives on it). # volume is still needed (inbound-api-keys.sqlite lives on it).
- ./central-node-b/data:/app/data - central-node-b-data:/app/data
# Same host directory, mounted a second time OUTSIDE the content root (/app). # Same named volume, mounted a second time OUTSIDE the content root (/app).
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is # Secrets 0.6.2 refuses a store path under the content root; this mount is
# not touched by an image rebuild, but the guard is lexical and cannot know that. # not touched by an image rebuild, but the guard is lexical and cannot know that.
- ./central-node-b/data:/data - central-node-b-data:/data
- ./central-node-b/logs:/app/logs - ./central-node-b/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -226,11 +226,11 @@ services:
- "9023:8083" # gRPC streaming - "9023:8083" # gRPC streaming
volumes: volumes:
- ./site-a-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro - ./site-a-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro
- ./site-a-node-a/data:/app/data - site-a-node-a-data:/app/data
# Same host directory, mounted a second time OUTSIDE the content root (/app). # Same named volume, mounted a second time OUTSIDE the content root (/app).
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is # Secrets 0.6.2 refuses a store path under the content root; this mount is
# not touched by an image rebuild, but the guard is lexical and cannot know that. # not touched by an image rebuild, but the guard is lexical and cannot know that.
- ./site-a-node-a/data:/data - site-a-node-a-data:/data
- ./site-a-node-a/logs:/app/logs - ./site-a-node-a/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -262,11 +262,11 @@ services:
- "9024:8083" # gRPC streaming - "9024:8083" # gRPC streaming
volumes: volumes:
- ./site-a-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro - ./site-a-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro
- ./site-a-node-b/data:/app/data - site-a-node-b-data:/app/data
# Same host directory, mounted a second time OUTSIDE the content root (/app). # Same named volume, mounted a second time OUTSIDE the content root (/app).
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is # Secrets 0.6.2 refuses a store path under the content root; this mount is
# not touched by an image rebuild, but the guard is lexical and cannot know that. # not touched by an image rebuild, but the guard is lexical and cannot know that.
- ./site-a-node-b/data:/data - site-a-node-b-data:/data
- ./site-a-node-b/logs:/app/logs - ./site-a-node-b/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -296,11 +296,11 @@ services:
- "9033:8083" # gRPC streaming - "9033:8083" # gRPC streaming
volumes: volumes:
- ./site-b-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro - ./site-b-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro
- ./site-b-node-a/data:/app/data - site-b-node-a-data:/app/data
# Same host directory, mounted a second time OUTSIDE the content root (/app). # Same named volume, mounted a second time OUTSIDE the content root (/app).
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is # Secrets 0.6.2 refuses a store path under the content root; this mount is
# not touched by an image rebuild, but the guard is lexical and cannot know that. # not touched by an image rebuild, but the guard is lexical and cannot know that.
- ./site-b-node-a/data:/data - site-b-node-a-data:/data
- ./site-b-node-a/logs:/app/logs - ./site-b-node-a/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -330,11 +330,11 @@ services:
- "9034:8083" # gRPC streaming - "9034:8083" # gRPC streaming
volumes: volumes:
- ./site-b-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro - ./site-b-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro
- ./site-b-node-b/data:/app/data - site-b-node-b-data:/app/data
# Same host directory, mounted a second time OUTSIDE the content root (/app). # Same named volume, mounted a second time OUTSIDE the content root (/app).
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is # Secrets 0.6.2 refuses a store path under the content root; this mount is
# not touched by an image rebuild, but the guard is lexical and cannot know that. # not touched by an image rebuild, but the guard is lexical and cannot know that.
- ./site-b-node-b/data:/data - site-b-node-b-data:/data
- ./site-b-node-b/logs:/app/logs - ./site-b-node-b/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -364,11 +364,11 @@ services:
- "9043:8083" # gRPC streaming - "9043:8083" # gRPC streaming
volumes: volumes:
- ./site-c-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro - ./site-c-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro
- ./site-c-node-a/data:/app/data - site-c-node-a-data:/app/data
# Same host directory, mounted a second time OUTSIDE the content root (/app). # Same named volume, mounted a second time OUTSIDE the content root (/app).
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is # Secrets 0.6.2 refuses a store path under the content root; this mount is
# not touched by an image rebuild, but the guard is lexical and cannot know that. # not touched by an image rebuild, but the guard is lexical and cannot know that.
- ./site-c-node-a/data:/data - site-c-node-a-data:/data
- ./site-c-node-a/logs:/app/logs - ./site-c-node-a/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -398,11 +398,11 @@ services:
- "9044:8083" # gRPC streaming - "9044:8083" # gRPC streaming
volumes: volumes:
- ./site-c-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro - ./site-c-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro
- ./site-c-node-b/data:/app/data - site-c-node-b-data:/app/data
# Same host directory, mounted a second time OUTSIDE the content root (/app). # Same named volume, mounted a second time OUTSIDE the content root (/app).
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is # Secrets 0.6.2 refuses a store path under the content root; this mount is
# not touched by an image rebuild, but the guard is lexical and cannot know that. # not touched by an image rebuild, but the guard is lexical and cannot know that.
- ./site-c-node-b/data:/data - site-c-node-b-data:/data
- ./site-c-node-b/logs:/app/logs - ./site-c-node-b/logs:/app/logs
networks: networks:
- scadabridge-net - scadabridge-net
@@ -424,3 +424,29 @@ services:
networks: networks:
scadabridge-net: scadabridge-net:
external: true external: true
# Named volumes for every node's SQLite data dir (register row 79). On macOS/OrbStack a
# bind mount crosses virtiofs, which does not give SQLite true fsync durability — both
# site-a LocalDb files corrupted through that layer (row 69). Named volumes live on
# native ext4 inside the OrbStack VM, taking virtiofs out of the write path entirely.
# Host-side inspection still works read-only via ~/OrbStack/docker/volumes/<name>;
# `docker compose down` preserves them; remove with `docker volume rm <name>`.
# The appsettings/logs bind mounts stay: config is ro, and Serilog's append-only files
# have no fsync-durability requirement worth losing `tail -f` from the repo tree for.
volumes:
central-node-a-data:
name: scadabridge-central-node-a-data
central-node-b-data:
name: scadabridge-central-node-b-data
site-a-node-a-data:
name: scadabridge-site-a-node-a-data
site-a-node-b-data:
name: scadabridge-site-a-node-b-data
site-b-node-a-data:
name: scadabridge-site-b-node-a-data
site-b-node-b-data:
name: scadabridge-site-b-node-b-data
site-c-node-a-data:
name: scadabridge-site-c-node-a-data
site-c-node-b-data:
name: scadabridge-site-c-node-b-data
+2 -2
View File
@@ -10,8 +10,8 @@ docker compose -f "$SCRIPT_DIR/docker-compose.yml" down
echo "" echo ""
echo "Teardown complete." echo "Teardown complete."
echo "Site data (SQLite DBs) and logs are preserved in node directories." echo "Node data (SQLite DBs) is preserved in named docker volumes; logs in node directories."
echo "" echo ""
echo "To also remove persistent data:" echo "To also remove persistent data:"
echo " rm -rf $SCRIPT_DIR/site-a-node-a/data $SCRIPT_DIR/site-a-node-b/data" echo " docker volume rm \$(docker volume ls -q --filter name=scadabridge-central-node --filter name=scadabridge-site-)"
echo " rm -rf $SCRIPT_DIR/*/logs" echo " rm -rf $SCRIPT_DIR/*/logs"
+39 -20
View File
@@ -359,18 +359,25 @@ database from its peer rather than letting it rejoin.
### Docker-rig data files on macOS/OrbStack — a virtiofs corruption + availability hazard ### Docker-rig data files on macOS/OrbStack — a virtiofs corruption + availability hazard
**Status: MITIGATED 2026-08-18 (register row 79) — the rigs' data dirs moved to named docker
volumes**, taking virtiofs out of the SQLite write path entirely. The hazard description and the
proven recovery runbook below are retained because they explain *why* the volumes exist, and
because failure mode 2 (the VM idle-sleep) is unaffected by the move.
**Scope: the local `docker/` and `docker-env2/` development rigs on macOS only.** This is a **Scope: the local `docker/` and `docker-env2/` development rigs on macOS only.** This is a
host-level property of the filesystem the rig's data files sit on, **not a ScadaBridge defect** and host-level property of the filesystem the rig's data files sit on, **not a ScadaBridge defect** and
not something a production Linux deployment is exposed to. It is documented here because it has not something a production Linux deployment is exposed to. It is documented here because it has
already cost two databases and half a session, and because the recovery is non-obvious. already cost two databases and half a session, and because the recovery is non-obvious.
Every ScadaBridge data file on the rig rides a **relative bind mount** out of the repo working tree Until 2026-08-18, every ScadaBridge data file on the rig rode a **relative bind mount** out of the
— `./site-a-node-a/data:/app/data` and its siblings, which is where `site-localdb.db`, repo working tree — `./site-a-node-a/data:/app/data` and its siblings, which is where
`auditlog.db` and the secrets store all live. On macOS that path crosses **virtiofs** to reach the `site-localdb.db`, `auditlog.db` and the secrets store all live. On macOS that path crosses
Linux VM, and **virtiofs does not give SQLite true `fsync` durability**. SQLite's crash-safety **virtiofs** to reach the Linux VM, and **virtiofs does not give SQLite true `fsync` durability**.
contract assumes a successful `fsync` means the bytes are on stable storage; when that is not true, SQLite's crash-safety contract assumes a successful `fsync` means the bytes are on stable storage;
an abrupt container kill mid-write can leave a torn file. `docker compose up --force-recreate` when that is not true, an abrupt container kill mid-write can leave a torn file.
— which `deploy.sh` uses — is exactly such an abrupt kill. `docker compose up --force-recreate` — which `deploy.sh` uses — is exactly such an abrupt kill.
Today the same paths are **named volumes** (`scadabridge-<node>-data`,
`scadabridge-env2-<node>-data`) on native ext4 inside the OrbStack VM, where `fsync` is real.
Two distinct failure modes have been observed live (both 2026-08-15): Two distinct failure modes have been observed live (both 2026-08-15):
@@ -421,13 +428,16 @@ be restored from a backup instead.
1. **Stop BOTH nodes of the site pair, together.** Not one at a time — this is the same 1. **Stop BOTH nodes of the site pair, together.** Not one at a time — this is the same
stop-and-start-together rule as *Site Pair Upgrades* above, for the same reason. stop-and-start-together rule as *Site Pair Upgrades* above, for the same reason.
2. **Quarantine the database and its sidecars — do not delete them.** Rename all three, keeping 2. **Quarantine the database and its sidecars — do not delete them.** Rename all three, keeping
them for forensics: them for forensics. The data dir now lives in the node's named volume, so run the renames
through a throwaway container (don't write through the host-side `~/OrbStack` view):
```sh ```sh
# per affected node's data directory # per affected node — volume name is scadabridge-<node>-data
mv site-localdb.db site-localdb.db.corrupt-$(date +%F) docker run --rm -v scadabridge-site-a-node-a-data:/d alpine sh -c '
mv site-localdb.db-wal site-localdb.db-wal.corrupt-$(date +%F) cd /d
mv site-localdb.db-shm site-localdb.db-shm.corrupt-$(date +%F) mv site-localdb.db site-localdb.db.corrupt-$(date +%F)
mv site-localdb.db-wal site-localdb.db-wal.corrupt-$(date +%F)
mv site-localdb.db-shm site-localdb.db-shm.corrupt-$(date +%F)'
``` ```
The `-wal` and `-shm` files **must** move with the main file. Leaving a stale WAL beside a fresh The `-wal` and `-shm` files **must** move with the main file. Leaving a stale WAL beside a fresh
@@ -443,16 +453,25 @@ On the live recovery, both quarantined files were large (394 MB and 416 MB), the
`connected: true` / backlog 0, and a schema change in flight at the time additionally proved the `connected: true` / backlog 0, and a schema change in flight at the time additionally proved the
digest handshake green. digest handshake green.
#### Mitigation options, with their trade-offs #### Mitigation applied (2026-08-18): named docker volumes
None of these is applied today; the rig deliberately still runs on bind mounts, and the accepted The "accept + runbook" position was retired without waiting for a second corruption. All ten
position is the runbook above. Recorded so the choice is a choice: data dirs (8 `docker/` nodes + 2 `docker-env2/` site nodes) now ride named volumes
(`scadabridge-<node>-data` / `scadabridge-env2-<node>-data`), declared in each compose file with
explicit `name:`s and seeded from the old bind-mount dirs at migration time. What changed the
original trade-off calculus: the assumed cost — losing host-side inspectability — turned out to
be much smaller than documented, because OrbStack exposes every volume on the Mac at
`~/OrbStack/docker/volumes/<name>/`, so `site-localdb.db` can still be opened from the
host for **read-only inspection**. (Never write through that path: it reintroduces exactly the
virtiofs layer the volumes exist to avoid.) The appsettings and logs bind mounts deliberately
stayed: config is read-only, and Serilog's append-only log files have no fsync-durability
requirement worth losing `tail -f` from the repo tree for.
| Option | Gains | Costs | Residual exposure: **failure mode 2 is availability-only and still real** — the VM can still
|--------|-------|-------| idle-sleep mid-operation and hang `docker` until `orb restart docker`; data now survives that
| Move rig data dirs to **named docker volumes** | Files live on native ext4 inside the VM, out of virtiofs entirely — removes the corruption mechanism at the source | Loses the host-side inspectability the debugging sessions lean on constantly: no more opening `site-localdb.db` from the Mac, no `ls` of a node's data dir from the repo tree. Requires a compose change. | on ext4. Disabling OrbStack's idle pause remains an un-version-controllable host setting and was
| **Disable OrbStack's idle pause** | Removes failure mode 2 (the mid-operation sleep and the virtiofs `refresh_nodeid` storm) | A host-level setting outside this repo, so it cannot be version-controlled or applied by `deploy.sh` — it silently does not exist on a new machine. Does nothing about failure mode 1. | left alone. The old `docker/<node>/data/` dirs were left in place as a dormant pre-migration
| **Accept, and use this runbook** (current) | No changes to compose or host config; full host-side inspectability retained | Corruption recurs on the next unlucky `--force-recreate`, and can sit latent until a later upgrade turns it into a crash loop. | backup; they are no longer mounted and can be deleted once the volumes have proven themselves.
### Central-Site Communication ### Central-Site Communication
@@ -71,7 +71,7 @@ All 7 fix-now items landed via PLAN-04/05/06/07/08 (verified in review 08 round
| 76 | ~~**CLI README's `audit-config query` JSON sample omits `afterStateJson`/`bundleImportId`** *(Low, docs)*~~ **RESOLVED 2026-08-16 (`4ae57bdb`).** Sample now shows both fields with their nullability semantics, and was grown to two entries mirroring the table-output sample below it row-for-row — which is what makes the "the table drops these two fields" sentence checkable. | Row-57 fix (2026-08-15) | Closed 2026-08-16. | — | | 76 | ~~**CLI README's `audit-config query` JSON sample omits `afterStateJson`/`bundleImportId`** *(Low, docs)*~~ **RESOLVED 2026-08-16 (`4ae57bdb`).** Sample now shows both fields with their nullability semantics, and was grown to two entries mirroring the table-output sample below it row-for-row — which is what makes the "the table drops these two fields" sentence checkable. | Row-57 fix (2026-08-15) | Closed 2026-08-16. | — |
| 77 | ~~**Test pattern: reflecting `ActorCell.Actor` right after `ResolveOne` is unordered vs child construction** *(Low, test-infra)*~~ **RESOLVED 2026-08-16 — swept clean, no commit needed.** Full `tests/` sweep: `ActorCell` (1 hit — the already-fixed file's comment), `ActorRefWithCell` (1 — fixed file :135), `.Underlying` (1 — same line), `BindingFlags` near actor/cell (2 — same file), `.Cell` (0), `ResolveOne` (3 files: the fixed one; `SiteAlarmLiveCacheServiceTests:432` resolves-then-`Sys.Stop`s, benign; `ReconcileStartsInstanceTests:363` uses the ref only for `IActorRef` equality, benign), `UnderlyingActor` (6 files — all Akka.TestKit `TestActorRef<T>` via `ActorOfAsTestActorRef`, an ORDERED mechanism, not this pattern). The unordered idiom exists in exactly one place and already carries the bounded-deadline retry. | Row-40 session (2026-08-15) | Closed 2026-08-16. | — | | 77 | ~~**Test pattern: reflecting `ActorCell.Actor` right after `ResolveOne` is unordered vs child construction** *(Low, test-infra)*~~ **RESOLVED 2026-08-16 — swept clean, no commit needed.** Full `tests/` sweep: `ActorCell` (1 hit — the already-fixed file's comment), `ActorRefWithCell` (1 — fixed file :135), `.Underlying` (1 — same line), `BindingFlags` near actor/cell (2 — same file), `.Cell` (0), `ResolveOne` (3 files: the fixed one; `SiteAlarmLiveCacheServiceTests:432` resolves-then-`Sys.Stop`s, benign; `ReconcileStartsInstanceTests:363` uses the ref only for `IActorRef` equality, benign), `UnderlyingActor` (6 files — all Akka.TestKit `TestActorRef<T>` via `ActorOfAsTestActorRef`, an ORDERED mechanism, not this pattern). The unordered idiom exists in exactly one place and already carries the bounded-deadline retry. | Row-40 session (2026-08-15) | Closed 2026-08-16. | — |
| 78 | ~~**Configs arriving via LocalDb replication start no InstanceActor — row 67's gap through a side door** *(Medium-Low, availability; proven live 2026-08-15)*~~ **RESOLVED 2026-08-16 (`e606b24a`, with row 72 — the DM instance sweep).** A periodic sweep on the Deployment Manager **singleton** (`ScadaBridge:SiteRuntime:InstanceSweepIntervalSeconds`, default 60, validated `>= 0`, `0` disables — default is live everywhere with no rig config change, which is intended): each tick re-reads the durable rows off-thread and, on the actor thread, starts any **enabled** stored config with no running actor. Arrival-path-agnostic by design — covers replication seeding, post-boot snapshot resyncs, and any future path. A LocalDb change hook was considered and rejected on evidence: it covers only the replication path and only the replicated half of the fleet (site-b/c aren't), AND `ILocalDb` 0.3.0 exposes no change-notification API at all, so a hook means a cross-repo upstream capability to close a site-local gap. Warning site event on a real heal names the sweep as the trigger; silent when there is nothing to do. Fail-before (sweep unarmed): 5 of 10 new tests fail including the exact live symptom ("A replication-seeded config for an enabled instance left no running Instance Actor"); 5 negative guards (disabled / in-flight-deploy deferral / matching-revision no-op / silence / interval-0) green in both states. Design note: `docs/plans/2026-08-15-instance-sweep-self-heal-design.md`. **LIVE-PROVEN at close-out (2026-08-16) in the exact original failure sequence:** site-a pair stopped, node-a's LocalDb file quarantined, pair restarted together — node-a's 08:45:55 reconcile reported "0 fetched" (replication had seeded the fresh store; central rightly saw no gap — the precise side door), and at 08:46:50, one sweep interval later, BOTH enabled instances started with the Warning `Instance rig-probe-1 started by deployment-manager sweep self-heal — its config is present in the deployed-config store but no Instance Actor was running until now`; `debug snapshot` live with no extra restart. The probe also exposed the DCL sibling gap — row 85. | This wave's close-out (2026-08-15) | Closed 2026-08-16. | — | | 78 | ~~**Configs arriving via LocalDb replication start no InstanceActor — row 67's gap through a side door** *(Medium-Low, availability; proven live 2026-08-15)*~~ **RESOLVED 2026-08-16 (`e606b24a`, with row 72 — the DM instance sweep).** A periodic sweep on the Deployment Manager **singleton** (`ScadaBridge:SiteRuntime:InstanceSweepIntervalSeconds`, default 60, validated `>= 0`, `0` disables — default is live everywhere with no rig config change, which is intended): each tick re-reads the durable rows off-thread and, on the actor thread, starts any **enabled** stored config with no running actor. Arrival-path-agnostic by design — covers replication seeding, post-boot snapshot resyncs, and any future path. A LocalDb change hook was considered and rejected on evidence: it covers only the replication path and only the replicated half of the fleet (site-b/c aren't), AND `ILocalDb` 0.3.0 exposes no change-notification API at all, so a hook means a cross-repo upstream capability to close a site-local gap. Warning site event on a real heal names the sweep as the trigger; silent when there is nothing to do. Fail-before (sweep unarmed): 5 of 10 new tests fail including the exact live symptom ("A replication-seeded config for an enabled instance left no running Instance Actor"); 5 negative guards (disabled / in-flight-deploy deferral / matching-revision no-op / silence / interval-0) green in both states. Design note: `docs/plans/2026-08-15-instance-sweep-self-heal-design.md`. **LIVE-PROVEN at close-out (2026-08-16) in the exact original failure sequence:** site-a pair stopped, node-a's LocalDb file quarantined, pair restarted together — node-a's 08:45:55 reconcile reported "0 fetched" (replication had seeded the fresh store; central rightly saw no gap — the precise side door), and at 08:46:50, one sweep interval later, BOTH enabled instances started with the Warning `Instance rig-probe-1 started by deployment-manager sweep self-heal — its config is present in the deployed-config store but no Instance Actor was running until now`; `debug snapshot` live with no extra restart. The probe also exposed the DCL sibling gap — row 85. | This wave's close-out (2026-08-15) | Closed 2026-08-16. | — |
| 79 | **OrbStack virtiofs is a live corruption + availability hazard for the rig's bind-mounted SQLite files** *(Medium, rig infra — host-level, not product)* — **mitigation chosen 2026-08-16: accept + document (`851b528c`).** The full hazard description + the proven quarantine-and-reseed recovery runbook now live in `docs/deployment/topology-guide.md` (new section), including the `orb restart docker` VM-sleep recovery and the named-volumes/idle-pause alternatives with their trade-offs. Row stays OPEN as the hazard tracker: a recurrence should trigger actually moving to named volumes or disabling idle-pause rather than re-running the runbook a third time. Original evidence: the OrbStack VM idle-SLEPT mid-session (`vmgr … msg=sleep`) leaving docker hung ~20 min until `orb restart docker`, with virtiofs `refresh_nodeid: race with new lookup` ERRORs on wake; both site-a LocalDb files (row 69) corrupted through this bind-mount layer — macOS virtiofs does not give SQLite true fsync durability, and abrupt container kills mid-write compound it. | This wave's close-out (2026-08-15); runbook shipped 2026-08-16 | Runbook shipped; hazard accepted for now | **Next corruption occurrence ⇒ stop accepting: move the rig data dirs to named volumes or disable OrbStack idle-pause** | | 79 | ~~**OrbStack virtiofs is a live corruption + availability hazard for the rig's bind-mounted SQLite files**~~ **RESOLVED 2026-08-18 — the prescribed remedy executed without waiting for a second corruption: all ten rig data dirs (8 `docker/` nodes + 2 `docker-env2/` site nodes) moved to named docker volumes** (`scadabridge-<node>-data` / `scadabridge-env2-<node>-data`, explicit `name:`s in both compose files), seeded from the old bind dirs with both rigs stopped, then redeployed and live-verified: site-a `localDbReplicationConnected: true` / backlog 0 over the migrated files, deployed instances resubscribed on boot (data continuity proven), env2 stable with 0 restarts. The trade-off that had deferred this — losing host-side inspectability — turned out to be mostly moot: OrbStack exposes every volume on the Mac at `~/OrbStack/docker/volumes/<name>/` (read-only inspection; never write through it). The rebuild also surfaced latent env2 drift: env2 was running a pre-Secrets-0.6.2 image, so on the fresh image all four env2 app nodes failed the new store-path validation at boot — fixed by porting the main rig's `x-secrets-store-env` posture (`Secrets__SqlitePath=/data/…` + `/data` second mounts, incl. NEW env2 central data volumes; env2 centrals previously kept data on the ephemeral overlay). Residual accepted: failure mode 2 (VM idle-sleep, `orb restart docker` recovery) is availability-only and unaffected — data now survives it on ext4. Hazard history + recovery runbook retained in `docs/deployment/topology-guide.md` (section updated). Old `docker/<node>/data/` dirs left in place as dormant pre-migration backups. | This wave's close-out (2026-08-15); runbook 2026-08-16; migration executed 2026-08-18 | Closed 2026-08-18. | — |
| 80 | ~~**Unreplicated sites report NON-NULL `LocalDbReplicationConnected: false` / `LocalDbOplogBacklog: 0` — contradicting the documented nullable semantics** *(Low-Medium, observability correctness)*~~ **RESOLVED 2026-08-16 (`a2f0086f`, with row 73).** Root cause was type-level: `ISiteHealthCollector.SetLocalDbReplicationStatus(bool …)` made null INEXPRESSIBLE for the connected flag, and the Host wired the bridge unconditionally — the replication engine is registered on every site node and idles answering `false`/`0` with no peer. Fix mirrors the `SiteEventLogReplicationCheck` precedent exactly: the Host evaluates `SiteLocalDbSetup.ReplicationIsConfigured` once and passes it as a third provider to `AddLocalDbReplicationHealthBridge` (the SAME captured value now also feeds `SiteEventLogReplicationCheck`, so the two surfaces can't disagree); unconfigured short-circuits to `(null, null)`; the collector widens to `bool?`. The rest of the chain needed NO change — proto already `BoolValue`/`Int64Value` wrappers, `NextBacklogAboveSince` already leaves the anchor unchanged on null, KPI source already conditional: row 33's machinery was correct and had simply never received a null (its null branch was dead code in production until now). `false` now means exactly one thing: configured, peer session down = real fault. Trap that produced the bug, now pinned in tests: configured-ness is `PeerAddress` **OR** `ApiKey` — "no PeerAddress" is NOT "unreplicated" (a passive-half node carries the key alone); the pre-existing Host fixture asserting `false`/`0` was exactly such a node and its assertion is CORRECT — kept with an explaining comment, plus a new genuinely-unreplicated fixture. Fail-before: gate behaviourally reverted → `UnreplicatedSite_HealthReport_ReportsNullForBothLocalDbFields` fails `Expected: null / Actual: False`. | Found at the previous wave's close-out verifying row 33 | Closed 2026-08-16. | — | | 80 | ~~**Unreplicated sites report NON-NULL `LocalDbReplicationConnected: false` / `LocalDbOplogBacklog: 0` — contradicting the documented nullable semantics** *(Low-Medium, observability correctness)*~~ **RESOLVED 2026-08-16 (`a2f0086f`, with row 73).** Root cause was type-level: `ISiteHealthCollector.SetLocalDbReplicationStatus(bool …)` made null INEXPRESSIBLE for the connected flag, and the Host wired the bridge unconditionally — the replication engine is registered on every site node and idles answering `false`/`0` with no peer. Fix mirrors the `SiteEventLogReplicationCheck` precedent exactly: the Host evaluates `SiteLocalDbSetup.ReplicationIsConfigured` once and passes it as a third provider to `AddLocalDbReplicationHealthBridge` (the SAME captured value now also feeds `SiteEventLogReplicationCheck`, so the two surfaces can't disagree); unconfigured short-circuits to `(null, null)`; the collector widens to `bool?`. The rest of the chain needed NO change — proto already `BoolValue`/`Int64Value` wrappers, `NextBacklogAboveSince` already leaves the anchor unchanged on null, KPI source already conditional: row 33's machinery was correct and had simply never received a null (its null branch was dead code in production until now). `false` now means exactly one thing: configured, peer session down = real fault. Trap that produced the bug, now pinned in tests: configured-ness is `PeerAddress` **OR** `ApiKey` — "no PeerAddress" is NOT "unreplicated" (a passive-half node carries the key alone); the pre-existing Host fixture asserting `false`/`0` was exactly such a node and its assertion is CORRECT — kept with an explaining comment, plus a new genuinely-unreplicated fixture. Fail-before: gate behaviourally reverted → `UnreplicatedSite_HealthReport_ReportsNullForBothLocalDbFields` fails `Expected: null / Actual: False`. | Found at the previous wave's close-out verifying row 33 | Closed 2026-08-16. | — |
| 81 | ~~**The site-event-log storage cap cannot be enforced on a replicated node with an undrained oplog** *(Medium-Low, measured 2026-08-16 during the row-34 work)*~~ **RESOLVED 2026-08-17 (`ddb3c32e`) — but NOT via the row's proposed `dbstat` fix, which is impossible on this stack.** Probed BEFORE implementing: the bundled SQLite 3.53.3 has no `ENABLE_DBSTAT_VTAB` (`pragma_module_list` returns 10 modules, none of them `dbstat`/`sqlite_dbpage`); pinned by `SqliteCapabilityTests`, written to FAIL if a future bump enables it (row 94). Switching SQLite builds rejected — the provider is shared with `ZB.MOM.WW.LocalDb` fleet-wide. The cap instead measures `site_events` + its four indexes via a sampled page-layout model: row width sampled 500 rows from each end per tick (`octet_length`), whole-page table cost (linear models under-estimate near-half-page rows by ~40%), clamped by whole-file logical size (a true upper bound). Index pages count against the cap (~34% of footprint — the 1 GB default was sized against a figure that included them). Measured model error −0.8%/+8.4%/+12.8%, biased high on purpose; cross-checks row 34's independent 393 B/row at 397.2. Guard semantics re-based: (1) unsampleable row width ⇒ refuse to trim (checked BEFORE the cap comparison); (2) the old "size did not decrease" guard now runs on the table-scoped estimate, so replication bookkeeping can no longer trip it — it now means exactly one thing: events arriving as fast as the trim removes them. Bytes-per-row held fixed per tick (re-sampling per batch would jitter the estimate through guard 2 — the same self-defeating measurement as the row itself). LocalDb bookkeeping explicitly NOT managed here (`MaxOplogRows`/tombstone retention own it); topology guide gained a volume-sizing subsection stating the honest budget. Fail-before: reverting to whole-file semantics fails 5 new tests and reproduces the exact original signature (rows destroyed, cap still unmet). SiteEventLogging 104/104, Host 520/520. No config keys added or renamed. Decision memo: `docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md`. Model limitations recorded as rows 92/93. | Row-34 measurement harness; §5 of the measurement doc (postscript appended) | Closed 2026-08-17. | — | | 81 | ~~**The site-event-log storage cap cannot be enforced on a replicated node with an undrained oplog** *(Medium-Low, measured 2026-08-16 during the row-34 work)*~~ **RESOLVED 2026-08-17 (`ddb3c32e`) — but NOT via the row's proposed `dbstat` fix, which is impossible on this stack.** Probed BEFORE implementing: the bundled SQLite 3.53.3 has no `ENABLE_DBSTAT_VTAB` (`pragma_module_list` returns 10 modules, none of them `dbstat`/`sqlite_dbpage`); pinned by `SqliteCapabilityTests`, written to FAIL if a future bump enables it (row 94). Switching SQLite builds rejected — the provider is shared with `ZB.MOM.WW.LocalDb` fleet-wide. The cap instead measures `site_events` + its four indexes via a sampled page-layout model: row width sampled 500 rows from each end per tick (`octet_length`), whole-page table cost (linear models under-estimate near-half-page rows by ~40%), clamped by whole-file logical size (a true upper bound). Index pages count against the cap (~34% of footprint — the 1 GB default was sized against a figure that included them). Measured model error −0.8%/+8.4%/+12.8%, biased high on purpose; cross-checks row 34's independent 393 B/row at 397.2. Guard semantics re-based: (1) unsampleable row width ⇒ refuse to trim (checked BEFORE the cap comparison); (2) the old "size did not decrease" guard now runs on the table-scoped estimate, so replication bookkeeping can no longer trip it — it now means exactly one thing: events arriving as fast as the trim removes them. Bytes-per-row held fixed per tick (re-sampling per batch would jitter the estimate through guard 2 — the same self-defeating measurement as the row itself). LocalDb bookkeeping explicitly NOT managed here (`MaxOplogRows`/tombstone retention own it); topology guide gained a volume-sizing subsection stating the honest budget. Fail-before: reverting to whole-file semantics fails 5 new tests and reproduces the exact original signature (rows destroyed, cap still unmet). SiteEventLogging 104/104, Host 520/520. No config keys added or renamed. Decision memo: `docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md`. Model limitations recorded as rows 92/93. | Row-34 measurement harness; §5 of the measurement doc (postscript appended) | Closed 2026-08-17. | — |
| 82 | ~~**`AuditLog.Tests` flakes under parallel full-solution runs — `MsSqlMigrationFixture` contention** *(Low, test-infra)*~~ **RESOLVED 2026-08-17 (`da23f2dc`) — three mechanisms, and the row's "per-run database name" prescription was already in place (the fixture has minted `ScadaBridgeAuditMigTest_<guid>` per class since it was written).** (1) The headline fingerprint was create+migrate SATURATION: nothing bounded how many fixtures ran `CREATE DATABASE` + the ~100-migration chain at once (measured peak: 45 concurrent test DBs / 19 blocked requests / 98 sessions on one container), pushing DDL past EF's default 30 s command timeout — every observed failure was `..ctor → MigrateAsync → SqlException 258 Execution Timeout Expired`, which xunit reports against every test in the class (hence "29 failures"). Fix: a machine-wide counting semaphore on `sp_getapplock` in `master` (`ScadaBridgeTestMigrate/0..N-1`, 4 slots, `SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS` override) held across create+migrate — the arbiter must live on the shared SERVER because the contention is cross-process; gate connection is deliberately UNPOOLED (a pooled one defers the session-lock release until reuse, stranding a slot); crashed process ⇒ SQL releases the lock; a gate deadline miss proceeds UNGATED so the throttle can never itself be a red. Timeouts raised to 300 s migrate / 120 s create-drop (bounded queue = busy but progressing); once-per-process sweep of orphaned `ScadaBridgeAuditMigTest_*` >2 h; connectivity probe retries 3× (a busy server misread as absent flips every MSSQL test to a green-looking Skip — worse than a red). (2)+(3) Two more flakes behind it: `PartitionPurgeTests` and `NotifyDispatcherAuditTrailTests` LEAKED 100 ms-tick actors (purge / outbox dispatcher) whose only stop was timeout-bounded `TestKit.Dispose` — a leaked dispatcher even delivered the NEXT test's notification through its own exhausted adapter, stranding it `Retrying`. Fixed with `Watch`/`Stop`/`ExpectTerminated` in the creating test. Evidence: 10× concurrent ConfigurationDatabase.Tests went from 10/10 processes failing (17–77 failures each) to 3 consecutive 10/10-green runs; the AuditLog 3×+1× mix from failing to 3 clean runs; standalone AuditLog 383/383, ConfigurationDatabase 395/395 (no runtime regression), SiteCallAudit 66/66 (inherits the fixture); zero orphan DBs or stuck app locks left. Host.Tests/PlaywrightTests hit the PERSISTENT DB on an already-serialized axis — untouched. Systemic residue recorded as rows 95/96. Doc: `docs/test_infra/test_infra_db.md`. | Found independently by two agents in the 2026-08-16 wave | Closed 2026-08-17. | — | | 82 | ~~**`AuditLog.Tests` flakes under parallel full-solution runs — `MsSqlMigrationFixture` contention** *(Low, test-infra)*~~ **RESOLVED 2026-08-17 (`da23f2dc`) — three mechanisms, and the row's "per-run database name" prescription was already in place (the fixture has minted `ScadaBridgeAuditMigTest_<guid>` per class since it was written).** (1) The headline fingerprint was create+migrate SATURATION: nothing bounded how many fixtures ran `CREATE DATABASE` + the ~100-migration chain at once (measured peak: 45 concurrent test DBs / 19 blocked requests / 98 sessions on one container), pushing DDL past EF's default 30 s command timeout — every observed failure was `..ctor → MigrateAsync → SqlException 258 Execution Timeout Expired`, which xunit reports against every test in the class (hence "29 failures"). Fix: a machine-wide counting semaphore on `sp_getapplock` in `master` (`ScadaBridgeTestMigrate/0..N-1`, 4 slots, `SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS` override) held across create+migrate — the arbiter must live on the shared SERVER because the contention is cross-process; gate connection is deliberately UNPOOLED (a pooled one defers the session-lock release until reuse, stranding a slot); crashed process ⇒ SQL releases the lock; a gate deadline miss proceeds UNGATED so the throttle can never itself be a red. Timeouts raised to 300 s migrate / 120 s create-drop (bounded queue = busy but progressing); once-per-process sweep of orphaned `ScadaBridgeAuditMigTest_*` >2 h; connectivity probe retries 3× (a busy server misread as absent flips every MSSQL test to a green-looking Skip — worse than a red). (2)+(3) Two more flakes behind it: `PartitionPurgeTests` and `NotifyDispatcherAuditTrailTests` LEAKED 100 ms-tick actors (purge / outbox dispatcher) whose only stop was timeout-bounded `TestKit.Dispose` — a leaked dispatcher even delivered the NEXT test's notification through its own exhausted adapter, stranding it `Retrying`. Fixed with `Watch`/`Stop`/`ExpectTerminated` in the creating test. Evidence: 10× concurrent ConfigurationDatabase.Tests went from 10/10 processes failing (17–77 failures each) to 3 consecutive 10/10-green runs; the AuditLog 3×+1× mix from failing to 3 clean runs; standalone AuditLog 383/383, ConfigurationDatabase 395/395 (no runtime regression), SiteCallAudit 66/66 (inherits the fixture); zero orphan DBs or stuck app locks left. Host.Tests/PlaywrightTests hit the PERSISTENT DB on an already-serialized axis — untouched. Systemic residue recorded as rows 95/96. Doc: `docs/test_infra/test_infra_db.md`. | Found independently by two agents in the 2026-08-16 wave | Closed 2026-08-17. | — |
@@ -84,9 +84,9 @@ All 7 fix-now items landed via PLAN-04/05/06/07/08 (verified in review 08 round
| 89 | ~~**Connection-only artifact deploy does not reach the peer node's DCL**~~ **RESOLVED 2026-08-17 (merge `6cfb6a79`, agent commit `c57e74e6`) — premise corrected TWICE, then fixed.** (1) The standby's DCL is **EMPTY**, not "serving the old endpoint": `DeploymentManagerActor` (role-scoped singleton) is the sole sender of `CreateConnectionCommand` and only populates its own node's `dcl-manager`. The real defect was **create-time and not replication-specific**: `_createdConnections` is per singleton *incarnation*, and `data_connection_definitions` had NO production reader — so any restart/failover rebuilt every connection from the flattened-config snapshots and silently discarded newer connection-only artifact deploys (identical loss on an unreplicated single-node restart). (2) Arbitration decided and shipped: **the definitions store wins wherever it has an opinion; flattened config is authoritative only for names the store never owned** (deploy-only sites unchanged — memo §7.4's objection). A `_connectionDefinitions` overlay refreshes from `ReadStoreSnapshotAsync` at boot AND every sweep tick, applied before instance creation; ping-pong killed by construction (every ensure path resolves the *winner* before hashing, so one config per hash slot). Fail-before: 5/8 new `DclConnectionArbitrationTests` fail with the overlay inert. Decision memo: `docs/plans/2026-08-17-dcl-connection-arbitration-decision.md` (records the accepted residuals: re-adds land at next instance start, not instantly — mirrors the removal's own "bad quality until redeploy" semantic, pinned by test). The discovery that a *live* connection actor is never reconfigured in place on ANY node is a separate pre-existing defect → row 97. **Live rig probe at wave close-out (post-redeploy):** the singleton node logged exactly ONE boot-time `Created DCL connection OPC PLC Simulator` with zero update churn across later sweep ticks (no ping-pong from the overlay), the standby stayed silent (topology as corrected), all three rig-probe-1 tags Good, site-a replication connected / backlog 0. | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — | | 89 | ~~**Connection-only artifact deploy does not reach the peer node's DCL**~~ **RESOLVED 2026-08-17 (merge `6cfb6a79`, agent commit `c57e74e6`) — premise corrected TWICE, then fixed.** (1) The standby's DCL is **EMPTY**, not "serving the old endpoint": `DeploymentManagerActor` (role-scoped singleton) is the sole sender of `CreateConnectionCommand` and only populates its own node's `dcl-manager`. The real defect was **create-time and not replication-specific**: `_createdConnections` is per singleton *incarnation*, and `data_connection_definitions` had NO production reader — so any restart/failover rebuilt every connection from the flattened-config snapshots and silently discarded newer connection-only artifact deploys (identical loss on an unreplicated single-node restart). (2) Arbitration decided and shipped: **the definitions store wins wherever it has an opinion; flattened config is authoritative only for names the store never owned** (deploy-only sites unchanged — memo §7.4's objection). A `_connectionDefinitions` overlay refreshes from `ReadStoreSnapshotAsync` at boot AND every sweep tick, applied before instance creation; ping-pong killed by construction (every ensure path resolves the *winner* before hashing, so one config per hash slot). Fail-before: 5/8 new `DclConnectionArbitrationTests` fail with the overlay inert. Decision memo: `docs/plans/2026-08-17-dcl-connection-arbitration-decision.md` (records the accepted residuals: re-adds land at next instance start, not instantly — mirrors the removal's own "bad quality until redeploy" semantic, pinned by test). The discovery that a *live* connection actor is never reconfigured in place on ANY node is a separate pre-existing defect → row 97. **Live rig probe at wave close-out (post-redeploy):** the singleton node logged exactly ONE boot-time `Created DCL connection OPC PLC Simulator` with zero update churn across later sweep ticks (no ping-pong from the overlay), the standby stayed silent (topology as corrected), all three rig-probe-1 tags Good, site-a replication connected / backlog 0. | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — |
| 90 | ~~**No removal path for a connection deleted on the peer**~~ **RESOLVED 2026-08-17 (merge `6cfb6a79`, agent commit `c57e74e6`), with row 89.** Both removal paths now share `RemoveConnectionCentrally` + an in-memory tombstone that `EnsureDclConnections` honors — which also closed a pre-existing bug where an artifact removal was **undone by the very next instance start**. The sweep detects a previously-definition-sourced row's disappearance via its own overlay bookkeeping. Accepted residuals, documented in the decision memo: tombstones are not durable across a restart (by decision — matches the long-standing restart-resurrection semantic), and a delete is undetectable on a node that never adopted the row (the stronger "non-empty table ⇒ complete set" inference would stop live connections on a guess; deliberately not taken). | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — | | 90 | ~~**No removal path for a connection deleted on the peer**~~ **RESOLVED 2026-08-17 (merge `6cfb6a79`, agent commit `c57e74e6`), with row 89.** Both removal paths now share `RemoveConnectionCentrally` + an in-memory tombstone that `EnsureDclConnections` honors — which also closed a pre-existing bug where an artifact removal was **undone by the very next instance start**. The sweep detects a previously-definition-sourced row's disappearance via its own overlay bookkeeping. Accepted residuals, documented in the decision memo: tombstones are not durable across a restart (by decision — matches the long-standing restart-resurrection semantic), and a delete is undetectable on a node that never adopted the row (the stronger "non-empty table ⇒ complete set" inference would stop live connections on a guess; deliberately not taken). | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — |
| 91 | ~~**`SiteScriptCompileCacheTests.GetOrAdd_SameCodeAndGlobals_ComputesOnce` is parallel-load fragile**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `26e3a1b9`) — row-74 pattern applied (exemplar `98c94588`), and wider than the named test.** Two habits: a per-test-instance guid key token (no other class can collide with or serve this test's entries, making `factoryCalls`/`Assert.Same` exact and owned) + one-sided bounds on the process-global counters (`>= baseline + n`; safe because all `Clear()` callers share the serialized collection). Four assertions changed, two got STRONGER (`GetOrAdd_SameCode_DifferentGlobals` had a second absolute pair; `Overflow_StaysWithinTheBound` gained an eviction probe over its own keys — a slack bound is not proof a sweep ran; its `ForeignEntrySlack = 512` is a judgement call, the eviction probe is the load-bearing assertion). Sibling absolutes in the same class and in `DeploymentWarmThenGateTests` fixed too. SiteRuntime.Tests run twice at full parallelism, 633/633 both. The underlying process-global `Clear()` cross-talk hazard remains → row 100. | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — | | 91 | ~~**`SiteScriptCompileCacheTests.GetOrAdd_SameCodeAndGlobals_ComputesOnce` is parallel-load fragile**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `26e3a1b9`) — row-74 pattern applied (exemplar `98c94588`), and wider than the named test.** Two habits: a per-test-instance guid key token (no other class can collide with or serve this test's entries, making `factoryCalls`/`Assert.Same` exact and owned) + one-sided bounds on the process-global counters (`>= baseline + n`; safe because all `Clear()` callers share the serialized collection). Four assertions changed, two got STRONGER (`GetOrAdd_SameCode_DifferentGlobals` had a second absolute pair; `Overflow_StaysWithinTheBound` gained an eviction probe over its own keys — a slack bound is not proof a sweep ran; its `ForeignEntrySlack = 512` is a judgement call, the eviction probe is the load-bearing assertion). Sibling absolutes in the same class and in `DeploymentWarmThenGateTests` fixed too. SiteRuntime.Tests run twice at full parallelism, 633/633 both. The underlying process-global `Clear()` cross-talk hazard remains → row 100. | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — |
| 92 | **The event-log cap's size model is sample-based and blind to mid-history width shifts** *(Low, accuracy limitation — documented in memo §3.1)* | Row-81 fix session (2026-08-17) | Row width is sampled 500 rows from each END of `site_events`; a stream whose row width changes sharply mid-history is mis-sized until the trim reaches that region. Self-correcting per tick (each trim exposes new tail rows to the sample), and the model's deliberate +8–13% high bias absorbs modest shifts, but a pathological width profile could over- or under-trim for several ticks. | First field report of the cap trimming visibly wide of its target, or any rework of the size model (see rows 93/94) | | 92 | **The event-log cap's size model is sample-based and blind to mid-history width shifts** *(Low, accuracy limitation — documented in memo §3.1)* | Row-81 fix session (2026-08-17) | Row width is sampled 500 rows from each END of `site_events`; a stream whose row width changes sharply mid-history is mis-sized until the trim reaches that region. Self-correcting per tick (each trim exposes new tail rows to the sample), and the model's deliberate +8–13% high bias absorbs modest shifts, but a pathological width profile could over- or under-trim for several ticks. **Reviewed 2026-08-18: accepted by design** — inherent to any sampled model, not closable by code short of the row-94 `dbstat` replacement; the row-93 schema pin does not (and cannot) cover it. | First field report of the cap trimming visibly wide of its target, or the row-94 `dbstat` replacement (which retires this row with the model) |
| 93 | **The cap model's page-layout constants are empirical for the CURRENT `site_events` schema** *(Low, maintenance trap)* | Row-81 fix session (2026-08-17) | Fill factors (0.78/0.75), cell overheads (16/8) and especially `IndexedColumnCount = 4` are calibrated to today's table + four indexes. A schema change — new column, new index — silently invalidates them; a fifth index in particular would be silently under-counted. Any `site_events` schema migration must re-run the memo §4 calibration and update the constants together. | Any schema change to `site_events` or its indexes | | 93 | ~~**The cap model's page-layout constants are empirical for the CURRENT `site_events` schema**~~ **RESOLVED 2026-08-18 — the silent trap is now loud.** `SiteEventsSchemaCalibrationTests` (SiteEventLogging.Tests) pins the exact shape the calibration assumed: the eight-column list the sampler's `octet_length` sum covers, the index set as exactly the four secondary indexes `IndexedColumnCount = 4` charges plus the PK autoindex (absorbed into the measured fill factors), and each index's single covered column. Any `site_events` schema migration — new column, fifth index, composite widening — now fails the suite with a message directing to the memo §4 recalibration and the constants + sampling-SQL update, instead of silently mis-sizing the cap. Suite green 110/110. | Row-81 fix session (2026-08-17); pinned by test 2026-08-18 | Closed 2026-08-18. | — |
| 94 | **`SqliteCapabilityTests` is a deliberate tripwire — do not "fix" it by weakening the assertion** *(Low, standing note)* | Row-81 fix session (2026-08-17) | The test FAILS BY DESIGN if a future SQLite bump ships `dbstat`/`sqlite_dbpage` support. The correct response at that point is to REPLACE the sampled page-layout model with exact `dbstat` measurement (retiring rows 92/93 with it), not to relax the test. The test's own comment says the same. | `SqliteCapabilityTests` fails after a SQLite/provider bump | | 94 | **`SqliteCapabilityTests` is a deliberate tripwire — do not "fix" it by weakening the assertion** *(Low, standing note)* | Row-81 fix session (2026-08-17) | The test FAILS BY DESIGN if a future SQLite bump ships `dbstat`/`sqlite_dbpage` support. The correct response at that point is to REPLACE the sampled page-layout model with exact `dbstat` measurement (retiring row 92 — and the row-93 schema pin `SiteEventsSchemaCalibrationTests`, added 2026-08-18, goes with the model too), not to relax the test. The test's own comment says the same. | `SqliteCapabilityTests` fails after a SQLite/provider bump |
| 95 | ~~**The leaked 100 ms-tick test-actor pattern is systemic — three more classes carry it**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `34a8079d`).** All three classes live in AuditLog.Tests (premise correction: none in SiteCallAudit.Tests). `Watch`/`Stop`/`ExpectTerminated` replicated from `da23f2dc` at every creation site: 11 in `AuditLogPurgeActorTests`, 10 in `SiteAuditReconciliationActorTests`, 2 in `OutageReconciliationTests`. Second premise correction: `OutageReconciliationTests` DID call `Sys.Stop` — but fire-and-forget AND from the last line of the `await using` block, so an in-flight tick could still be writing to MSSQL during read-back and any earlier assertion failure skipped the stop entirely; stops moved before the read-backs, plus the `AuditLogIngestActor` that test 2 was discarding outright (holding a disposed provider) is now stopped too. Known lesser sibling habit left in place: EventStream probes are never unsubscribed (bounded by the per-test ActorSystem — noted, not a row). AuditLog.Tests 383/383. | Row-82 fix session (2026-08-17) | Closed 2026-08-17. | — | | 95 | ~~**The leaked 100 ms-tick test-actor pattern is systemic — three more classes carry it**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `34a8079d`).** All three classes live in AuditLog.Tests (premise correction: none in SiteCallAudit.Tests). `Watch`/`Stop`/`ExpectTerminated` replicated from `da23f2dc` at every creation site: 11 in `AuditLogPurgeActorTests`, 10 in `SiteAuditReconciliationActorTests`, 2 in `OutageReconciliationTests`. Second premise correction: `OutageReconciliationTests` DID call `Sys.Stop` — but fire-and-forget AND from the last line of the `await using` block, so an in-flight tick could still be writing to MSSQL during read-back and any earlier assertion failure skipped the stop entirely; stops moved before the read-backs, plus the `AuditLogIngestActor` that test 2 was discarding outright (holding a disposed provider) is now stopped too. Known lesser sibling habit left in place: EventStream probes are never unsubscribed (bounded by the per-test ActorSystem — noted, not a row). AuditLog.Tests 383/383. | Row-82 fix session (2026-08-17) | Closed 2026-08-17. | — |
| 96 | ~~**xunit parallelism is uncapped almost everywhere**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `d2d4db49`) — policy made and shipped: per-suite caps for the resource-heavy suites, pure unit suites stay uncapped.** The four MSSQL-fixture users (`ConfigurationDatabase.Tests`, `AuditLog.Tests`, `SiteCallAudit.Tests`, `Host.Tests` — verified before capping) get `xunit.runner.json` with `maxParallelThreads: 4`, **deliberately matching row-82's `sp_getapplock` slot default** so the in-process and cross-process limits agree; no collections serialized. `IntegrationTests` (already fully serialized) and `CentralUI.PlaywrightTests` (already `maxParallelThreads: 1`, stricter than needed) left alone. Cap verified live (216% CPU uncapped vs 145% capped on ConfigurationDatabase.Tests); cost is nil — these suites are latency-bound on MSSQL/sockets (14–25% CPU), durations unchanged within noise. Policy documented in `docs/test_infra/test_infra.md` → *Test Parallelism Policy* (two-axis model, per-suite table, default for new projects), cross-linked from `test_infra_db.md`. Found-not-fixed: `StoreAndForward.Tests/xunit.runner.json` is a no-op (pins the default `parallelizeTestCollections: true`) — harmless, tidy on next touch. | Row-82 fix session (2026-08-17) | Closed 2026-08-17. | — | | 96 | ~~**xunit parallelism is uncapped almost everywhere**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `d2d4db49`) — policy made and shipped: per-suite caps for the resource-heavy suites, pure unit suites stay uncapped.** The four MSSQL-fixture users (`ConfigurationDatabase.Tests`, `AuditLog.Tests`, `SiteCallAudit.Tests`, `Host.Tests` — verified before capping) get `xunit.runner.json` with `maxParallelThreads: 4`, **deliberately matching row-82's `sp_getapplock` slot default** so the in-process and cross-process limits agree; no collections serialized. `IntegrationTests` (already fully serialized) and `CentralUI.PlaywrightTests` (already `maxParallelThreads: 1`, stricter than needed) left alone. Cap verified live (216% CPU uncapped vs 145% capped on ConfigurationDatabase.Tests); cost is nil — these suites are latency-bound on MSSQL/sockets (14–25% CPU), durations unchanged within noise. Policy documented in `docs/test_infra/test_infra.md` → *Test Parallelism Policy* (two-axis model, per-suite table, default for new projects), cross-linked from `test_infra_db.md`. Found-not-fixed: `StoreAndForward.Tests/xunit.runner.json` is a no-op (pins the default `parallelizeTestCollections: true`) — harmless, tidy on next touch. | Row-82 fix session (2026-08-17) | Closed 2026-08-17. | — |
| 97 | ~~**A live DCL connection actor is never reconfigured in place — on ANY node**~~ **RESOLVED 2026-08-17 (merge `9dbda4a2`, agent commit `7c1268d8`) — the memo's §6 sketch shipped, plus two things the sketch missed.** `EnsureDclConnection` now has three outcomes off its one hash slot: unchanged → nothing; new → create; **changed → `ReconfigureConnectionCommand`** (arbitration/tombstones byte-for-byte untouched). The manager forwards to the owning actor (unknown name falls back to create); `DataConnectionActor` handles it in **all three** lifecycle states and never stashes it, via a new shared `SwapAdapter` helper now also used by both failover sites. The sketch's two gaps, found in implementation: **endpoint selection must be re-armed to Primary** (a connection sitting on its old *backup* would otherwise dial the new backup first), and the deliberate link drop must **suppress the unstable-disconnect accounting** (else the swap itself could trip a failover). Also a premise-sharpening: "reconfigure while mid-connect" is the *primary* case, not an edge — a wrong endpoint loops in `Connecting` forever and never reaches `Connected`. Fail-before: 5/7 new `DataConnectionReconfigureTests` + 3 SiteRuntime ensure-path tests fail on the reverted behavior. **Live rig probe (post-redeploy, round trip):** connection 1's config changed (operationTimeoutMs 15000→16000→back) fired exactly TWO in-place reconfigures on the active node — `Reconfiguring DataConnectionActor for OPC PLC Simulator in place` → `Reconnected successfully on Primary endpoint` in 45 ms → `Re-subscribing 2 tags` — with all rig-probe-1 tags Good after each, the standby silent (0 reconfigure lines, topology as designed), and the old dead-end "has a new configuration" Warning never appearing. Rig config restored to original. | Found by the 2026-08-17 rows-89/90 fix session; memo `docs/plans/2026-08-17-dcl-connection-arbitration-decision.md` (§9 postscript appended) | Closed 2026-08-17. | — | | 97 | ~~**A live DCL connection actor is never reconfigured in place — on ANY node**~~ **RESOLVED 2026-08-17 (merge `9dbda4a2`, agent commit `7c1268d8`) — the memo's §6 sketch shipped, plus two things the sketch missed.** `EnsureDclConnection` now has three outcomes off its one hash slot: unchanged → nothing; new → create; **changed → `ReconfigureConnectionCommand`** (arbitration/tombstones byte-for-byte untouched). The manager forwards to the owning actor (unknown name falls back to create); `DataConnectionActor` handles it in **all three** lifecycle states and never stashes it, via a new shared `SwapAdapter` helper now also used by both failover sites. The sketch's two gaps, found in implementation: **endpoint selection must be re-armed to Primary** (a connection sitting on its old *backup* would otherwise dial the new backup first), and the deliberate link drop must **suppress the unstable-disconnect accounting** (else the swap itself could trip a failover). Also a premise-sharpening: "reconfigure while mid-connect" is the *primary* case, not an edge — a wrong endpoint loops in `Connecting` forever and never reaches `Connected`. Fail-before: 5/7 new `DataConnectionReconfigureTests` + 3 SiteRuntime ensure-path tests fail on the reverted behavior. **Live rig probe (post-redeploy, round trip):** connection 1's config changed (operationTimeoutMs 15000→16000→back) fired exactly TWO in-place reconfigures on the active node — `Reconfiguring DataConnectionActor for OPC PLC Simulator in place` → `Reconnected successfully on Primary endpoint` in 45 ms → `Re-subscribing 2 tags` — with all rig-probe-1 tags Good after each, the standby silent (0 reconfigure lines, topology as designed), and the old dead-end "has a new configuration" Warning never appearing. Rig config restored to original. | Found by the 2026-08-17 rows-89/90 fix session; memo `docs/plans/2026-08-17-dcl-connection-arbitration-decision.md` (§9 postscript appended) | Closed 2026-08-17. | — |
@@ -0,0 +1,105 @@
using Microsoft.Data.Sqlite;
namespace ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests;
/// <summary>
/// Pins the <c>site_events</c> schema to the shape the storage-cap size model was calibrated
/// against (register row 93, <c>docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md</c> §4).
/// </summary>
/// <remarks>
/// <see cref="EventLogPurgeService.EstimateSiteEventsSize"/> hardcodes this schema twice over:
/// the sampling SQL sums <c>octet_length</c> over the exact column list, and
/// <c>IndexedColumnCount = 4</c> plus the fill-factor/overhead constants are empirical for
/// exactly these four secondary indexes. A schema migration that adds a column or an index
/// invalidates both SILENTLY — the new column's bytes are never sampled, a fifth index is
/// never charged — so this test turns that drift into a loud failure. The fix on failure is
/// NEVER to just update the assertions: re-run the memo §4 calibration, update the
/// <see cref="EventLogPurgeService"/> constants and sampling SQL together, and only then
/// re-pin the schema here.
/// </remarks>
public class SiteEventsSchemaCalibrationTests
{
private const string RecalibrationNotice =
"site_events schema drifted from the size-model calibration. Re-run the calibration in " +
"docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md §4 and update " +
"EventLogPurgeService's constants + sampling SQL together with this pin (register row 93).";
private static SqliteConnection OpenWithSchema()
{
var connection = new SqliteConnection("Data Source=:memory:");
connection.Open();
SiteEventLogSchema.Apply(connection);
return connection;
}
private static List<string> ReadStrings(SqliteConnection connection, string sql)
{
using var cmd = connection.CreateCommand();
cmd.CommandText = sql;
var values = new List<string>();
using var reader = cmd.ExecuteReader();
while (reader.Read())
values.Add(reader.GetString(0));
return values;
}
[Fact]
public void Columns_MatchTheSampledOctetLengthSum_Exactly()
{
// The sampler's payload_bytes expression sums octet_length over exactly these eight
// columns, in this order. A ninth column would contribute real disk bytes that the
// model never counts, under-sizing every estimate.
using var connection = OpenWithSchema();
var columns = ReadStrings(connection, "SELECT name FROM pragma_table_info('site_events')");
Assert.True(
columns.SequenceEqual(
["id", "timestamp", "event_type", "severity", "instance_id", "source", "message", "details"]),
$"{RecalibrationNotice} Actual columns: {string.Join(", ", columns)}");
}
[Fact]
public void Indexes_AreExactlyTheFourTheModelCharges_PlusThePkAutoindex()
{
// IndexedColumnCount = 4 charges four secondary index entries per row; the TEXT
// primary key's autoindex is absorbed into the empirically-measured fill factors.
// A fifth secondary index would be silently under-counted.
using var connection = OpenWithSchema();
var indexes = ReadStrings(
connection, "SELECT name FROM pragma_index_list('site_events') ORDER BY name");
Assert.True(
indexes.SequenceEqual(
[
"idx_events_instance",
"idx_events_severity",
"idx_events_timestamp",
"idx_events_type",
"sqlite_autoindex_site_events_1",
]),
$"{RecalibrationNotice} Actual indexes: {string.Join(", ", indexes)}");
}
[Theory]
[InlineData("idx_events_timestamp", "timestamp")]
[InlineData("idx_events_type", "event_type")]
[InlineData("idx_events_instance", "instance_id")]
[InlineData("idx_events_severity", "severity")]
public void EachIndex_CoversExactlyTheColumnTheSamplerSums(string indexName, string expectedColumn)
{
// The sampler's indexed_bytes expression sums octet_length(timestamp) +
// octet_length(event_type) + octet_length(severity) + octet_length(instance_id) —
// one column per index. Widening an index to a composite (or re-pointing it) would
// change the real entry width without changing what is sampled.
using var connection = OpenWithSchema();
var columns = ReadStrings(
connection, $"SELECT name FROM pragma_index_info('{indexName}')");
Assert.True(
columns.SequenceEqual([expectedColumn]),
$"{RecalibrationNotice} Index {indexName} covers: {string.Join(", ", columns)}");
}
}