infra(docker)+test(siteeventlog): close register tripwires 79 + 93 — rig data to named volumes; pin site_events schema to the cap-model calibration
Row 79 executed its own prescribed remedy without waiting for a second corruption: all ten rig data dirs (8 docker/ nodes + 2 docker-env2/ site nodes) move from virtiofs bind mounts to named docker volumes (scadabridge-<node>-data), seeded in place with both rigs stopped and live-verified (site-a localDbReplicationConnected=true / backlog 0 over the migrated files; env2 stable, 0 restarts). The feared inspectability loss is moot — OrbStack exposes volumes at ~/OrbStack/docker/volumes/. The rebuild surfaced latent env2 drift (pre-Secrets-0.6.2 image): env2 now carries the x-secrets-store-env posture + central data volumes. Row 93's silent maintenance trap is now loud: SiteEventsSchemaCalibrationTests pins the site_events columns/indexes the EventLogPurgeService size model is calibrated against, failing with a pointer to the memo §4 recalibration on any schema drift. Rows 92 (accepted by design) and 94 (standing tripwire) annotated accordingly.
This commit is contained in:
Joseph Doherty
@@ -1,3 +1,13 @@
|
||||
# Secrets 0.6.2 requires Secrets:SqlitePath to be absolute AND outside the content root
|
||||
# (/app) — mirrors docker/docker-compose.yml's x-secrets-store-env anchor, including the
|
||||
# trap documented there: DELETING the key does not fall back to the per-user default, it
|
||||
# yields the RELATIVE "secrets.db", which fails both rules and the node refuses to boot.
|
||||
# (env2 ran a stale pre-0.6.2 image until the 2026-08-18 rebuild, which surfaced this.)
|
||||
# /data is the node's own data volume mounted a second time outside /app, so the store
|
||||
# shares the durable volume without living under the content root.
|
||||
x-secrets-store-env: &secrets-store-env
|
||||
Secrets__SqlitePath: "/data/scadabridge-secrets.db"
|
||||
|
||||
services:
|
||||
central-a:
|
||||
image: scadabridge:latest
|
||||
@@ -7,6 +17,7 @@ services:
|
||||
stop_grace_period: 30s
|
||||
container_name: scadabridge-env2-central-a
|
||||
environment:
|
||||
<<: *secrets-store-env
|
||||
SCADABRIDGE_CONFIG: Central
|
||||
ASPNETCORE_ENVIRONMENT: Development
|
||||
ASPNETCORE_URLS: "http://+:5000"
|
||||
@@ -25,6 +36,10 @@ services:
|
||||
- "9111:8081" # Akka remoting
|
||||
volumes:
|
||||
- ./central-node-a/appsettings.Central.json:/app/appsettings.Central.json:ro
|
||||
- central-node-a-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app) for the
|
||||
# secrets store — see the x-secrets-store-env anchor above.
|
||||
- central-node-a-data:/data
|
||||
- ./central-node-a/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -38,6 +53,7 @@ services:
|
||||
stop_grace_period: 30s
|
||||
container_name: scadabridge-env2-central-b
|
||||
environment:
|
||||
<<: *secrets-store-env
|
||||
SCADABRIDGE_CONFIG: Central
|
||||
ASPNETCORE_ENVIRONMENT: Development
|
||||
ASPNETCORE_URLS: "http://+:5000"
|
||||
@@ -56,6 +72,10 @@ services:
|
||||
- "9112:8081" # Akka remoting
|
||||
volumes:
|
||||
- ./central-node-b/appsettings.Central.json:/app/appsettings.Central.json:ro
|
||||
- central-node-b-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app) for the
|
||||
# secrets store — see the x-secrets-store-env anchor above.
|
||||
- central-node-b-data:/data
|
||||
- ./central-node-b/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -69,6 +89,7 @@ services:
|
||||
stop_grace_period: 30s
|
||||
container_name: scadabridge-env2-site-x-a
|
||||
environment:
|
||||
<<: *secrets-store-env
|
||||
SCADABRIDGE_CONFIG: Site
|
||||
# Register row 63 — mirrors docker/docker-compose.yml. Enables the Development-only
|
||||
# DI scope validation (register row 50) on the site nodes, which previously ran
|
||||
@@ -79,7 +100,10 @@ services:
|
||||
- "9123:8083" # gRPC streaming
|
||||
volumes:
|
||||
- ./site-x-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro
|
||||
- ./site-x-node-a/data:/app/data
|
||||
- site-x-node-a-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app) for the
|
||||
# secrets store — see the x-secrets-store-env anchor above.
|
||||
- site-x-node-a-data:/data
|
||||
- ./site-x-node-a/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -93,6 +117,7 @@ services:
|
||||
stop_grace_period: 30s
|
||||
container_name: scadabridge-env2-site-x-b
|
||||
environment:
|
||||
<<: *secrets-store-env
|
||||
SCADABRIDGE_CONFIG: Site
|
||||
# Register row 63 — mirrors docker/docker-compose.yml. Enables the Development-only
|
||||
# DI scope validation (register row 50) on the site nodes, which previously ran
|
||||
@@ -103,7 +128,10 @@ services:
|
||||
- "9124:8083" # gRPC streaming
|
||||
volumes:
|
||||
- ./site-x-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro
|
||||
- ./site-x-node-b/data:/app/data
|
||||
- site-x-node-b-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app) for the
|
||||
# secrets store — see the x-secrets-store-env anchor above.
|
||||
- site-x-node-b-data:/data
|
||||
- ./site-x-node-b/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -125,3 +153,19 @@ services:
|
||||
networks:
|
||||
scadabridge-net:
|
||||
external: true
|
||||
|
||||
# Named volumes for the nodes' SQLite data dirs — mirrors docker/docker-compose.yml
|
||||
# (register row 79: bind mounts cross OrbStack's virtiofs, which does not give SQLite
|
||||
# true fsync durability). Host-side inspection via ~/OrbStack/docker/volumes/<name>.
|
||||
# The central volumes are NEW with the Secrets 0.6.2 posture — env2 centrals previously
|
||||
# kept data on the container overlay (lost on recreate), which 0.6.2's store-path guard
|
||||
# exists to prevent.
|
||||
volumes:
|
||||
central-node-a-data:
|
||||
name: scadabridge-env2-central-node-a-data
|
||||
central-node-b-data:
|
||||
name: scadabridge-env2-central-node-b-data
|
||||
site-x-node-a-data:
|
||||
name: scadabridge-env2-site-x-node-a-data
|
||||
site-x-node-b-data:
|
||||
name: scadabridge-env2-site-x-node-b-data
|
||||
|
||||
+21
-15
@@ -97,30 +97,29 @@ docker/
|
||||
│ └── logs/
|
||||
├── site-a-node-a/
|
||||
│ ├── appsettings.Site.json # Site-A node A configuration
|
||||
│ ├── data/ # SQLite databases (gitignored)
|
||||
│ └── logs/
|
||||
│ └── logs/ # Serilog file output (gitignored)
|
||||
├── site-a-node-b/
|
||||
│ ├── appsettings.Site.json
|
||||
│ ├── data/
|
||||
│ └── logs/
|
||||
├── site-b-node-a/
|
||||
│ ├── appsettings.Site.json # Site-B node A configuration
|
||||
│ ├── data/
|
||||
│ └── logs/
|
||||
├── site-b-node-b/
|
||||
│ ├── appsettings.Site.json
|
||||
│ ├── data/
|
||||
│ └── logs/
|
||||
├── site-c-node-a/
|
||||
│ ├── appsettings.Site.json # Site-C node A configuration
|
||||
│ ├── data/
|
||||
│ └── logs/
|
||||
└── site-c-node-b/
|
||||
├── appsettings.Site.json
|
||||
├── data/
|
||||
└── logs/
|
||||
```
|
||||
|
||||
SQLite databases live in **named docker volumes** (`scadabridge-<node>-data`, one per node —
|
||||
register row 79), not in the node directories; inspect them from the host (read-only) at
|
||||
`~/OrbStack/docker/volumes/<name>/`. Any `docker/<node>/data/` directory still on disk is a
|
||||
dormant pre-migration (2026-08-18) backup, no longer mounted.
|
||||
|
||||
## gRPC control-plane keys (dev)
|
||||
|
||||
The site gRPC service (`SiteStreamService` on 8083 — live subscriptions, audit pull,
|
||||
@@ -315,20 +314,27 @@ dotnet run --project src/ZB.MOM.WW.ScadaBridge.CLI -- \
|
||||
|
||||
### Clear Site Data
|
||||
|
||||
Remove SQLite databases to reset site state (deployed configs, S&F buffers):
|
||||
Remove SQLite databases to reset site state (deployed configs, S&F buffers). Data lives in
|
||||
named docker volumes (`scadabridge-<node>-data`, register row 79 — no longer bind-mounted
|
||||
under `docker/<node>/data`), so empty the volume with a throwaway container while the pair
|
||||
is stopped:
|
||||
|
||||
```bash
|
||||
# Single site
|
||||
rm -rf docker/site-a-node-a/data docker/site-a-node-b/data
|
||||
docker restart scadabridge-site-a-a scadabridge-site-a-b
|
||||
docker compose -f docker/docker-compose.yml stop site-a-a site-a-b
|
||||
docker run --rm \
|
||||
-v scadabridge-site-a-node-a-data:/a -v scadabridge-site-a-node-b-data:/b \
|
||||
alpine sh -c 'rm -rf /a/* /b/*'
|
||||
docker compose -f docker/docker-compose.yml start site-a-a site-a-b
|
||||
|
||||
# All sites
|
||||
rm -rf docker/site-*/data
|
||||
docker restart scadabridge-site-a-a scadabridge-site-a-b \
|
||||
scadabridge-site-b-a scadabridge-site-b-b \
|
||||
scadabridge-site-c-a scadabridge-site-c-b
|
||||
# All sites: repeat with the six scadabridge-site-*-data volumes, or teardown + volume rm
|
||||
# (see teardown.sh's hint) and redeploy.
|
||||
```
|
||||
|
||||
To inspect a node's live databases from the host, read them via OrbStack's volume
|
||||
exposure (read-only inspection — don't write through this path):
|
||||
`~/OrbStack/docker/volumes/scadabridge-site-a-node-a-data/site-localdb.db`.
|
||||
|
||||
### Rebuild Image From Scratch (no cache)
|
||||
|
||||
```bash
|
||||
|
||||
+6
-5
@@ -3,13 +3,14 @@ set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
# The compose file's bind mounts are RELATIVE (./site-a-node-a/data etc.), so running
|
||||
# this script from a git worktree silently rehomes every node's data directory to the
|
||||
# worktree's copy — the rig keeps running, but its SQLite stores split from the main
|
||||
# checkout's and all state written there is lost on the next deploy from main.
|
||||
# The compose file's remaining bind mounts (appsettings, logs) are RELATIVE, so running
|
||||
# this script from a git worktree silently rehomes every node's config to the worktree's
|
||||
# copy — the rig keeps running, but on stale/divergent appsettings. (The data dirs moved
|
||||
# to named volumes — register row 79 — so rig state itself no longer splits, but a
|
||||
# worktree deploy is still wrong by default.)
|
||||
if [[ "$SCRIPT_DIR" == *"/.claude/worktrees/"* && "${SCADABRIDGE_ALLOW_WORKTREE_DEPLOY:-}" != "1" ]]; then
|
||||
echo "ERROR: refusing to deploy from a git worktree ($SCRIPT_DIR)." >&2
|
||||
echo "Bind-mount data dirs would rehome to the worktree copy and split rig state." >&2
|
||||
echo "Bind-mounted appsettings would rehome to the worktree copy." >&2
|
||||
echo "Deploy from the main checkout, or set SCADABRIDGE_ALLOW_WORKTREE_DEPLOY=1 to override." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
+61
-35
@@ -6,9 +6,9 @@
|
||||
# The value must be absolute AND outside the content root (/app) — Secrets 0.6.2 validates
|
||||
# both at startup, because a store inside the deployment directory is destroyed by an
|
||||
# ordinary upgrade (this wiped the MxGateway key store on 2026-08-09). /data is the node's
|
||||
# own bind mount, the SAME host directory as /app/data, so an existing store is found
|
||||
# unmoved; only the path used to reach it changes. Do NOT point this back at /app/data —
|
||||
# absolute is not sufficient, and the boot will fail.
|
||||
# own data volume (named volume since register row 79), the SAME volume as /app/data, so an
|
||||
# existing store is found unmoved; only the path used to reach it changes. Do NOT point
|
||||
# this back at /app/data — absolute is not sufficient, and the boot will fail.
|
||||
#
|
||||
# Do NOT delete this key expecting the library's per-user default to take over. Measured on
|
||||
# this rig: with a Secrets section present but no SqlitePath, the effective value is the
|
||||
@@ -129,11 +129,11 @@ services:
|
||||
# central store moved to the shared SQL-Server database (Secrets 0.5.0,
|
||||
# scadaproj#4) the scadabridge-secrets.db here is a pre-0.5.0 residue, but the
|
||||
# volume is still needed (inbound-api-keys.sqlite lives on it).
|
||||
- ./central-node-a/data:/app/data
|
||||
# Same host directory, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is
|
||||
- central-node-a-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this mount is
|
||||
# not touched by an image rebuild, but the guard is lexical and cannot know that.
|
||||
- ./central-node-a/data:/data
|
||||
- central-node-a-data:/data
|
||||
- ./central-node-a/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -190,11 +190,11 @@ services:
|
||||
# central store moved to the shared SQL-Server database (Secrets 0.5.0,
|
||||
# scadaproj#4) the scadabridge-secrets.db here is a pre-0.5.0 residue, but the
|
||||
# volume is still needed (inbound-api-keys.sqlite lives on it).
|
||||
- ./central-node-b/data:/app/data
|
||||
# Same host directory, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is
|
||||
- central-node-b-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this mount is
|
||||
# not touched by an image rebuild, but the guard is lexical and cannot know that.
|
||||
- ./central-node-b/data:/data
|
||||
- central-node-b-data:/data
|
||||
- ./central-node-b/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -226,11 +226,11 @@ services:
|
||||
- "9023:8083" # gRPC streaming
|
||||
volumes:
|
||||
- ./site-a-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro
|
||||
- ./site-a-node-a/data:/app/data
|
||||
# Same host directory, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is
|
||||
- site-a-node-a-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this mount is
|
||||
# not touched by an image rebuild, but the guard is lexical and cannot know that.
|
||||
- ./site-a-node-a/data:/data
|
||||
- site-a-node-a-data:/data
|
||||
- ./site-a-node-a/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -262,11 +262,11 @@ services:
|
||||
- "9024:8083" # gRPC streaming
|
||||
volumes:
|
||||
- ./site-a-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro
|
||||
- ./site-a-node-b/data:/app/data
|
||||
# Same host directory, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is
|
||||
- site-a-node-b-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this mount is
|
||||
# not touched by an image rebuild, but the guard is lexical and cannot know that.
|
||||
- ./site-a-node-b/data:/data
|
||||
- site-a-node-b-data:/data
|
||||
- ./site-a-node-b/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -296,11 +296,11 @@ services:
|
||||
- "9033:8083" # gRPC streaming
|
||||
volumes:
|
||||
- ./site-b-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro
|
||||
- ./site-b-node-a/data:/app/data
|
||||
# Same host directory, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is
|
||||
- site-b-node-a-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this mount is
|
||||
# not touched by an image rebuild, but the guard is lexical and cannot know that.
|
||||
- ./site-b-node-a/data:/data
|
||||
- site-b-node-a-data:/data
|
||||
- ./site-b-node-a/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -330,11 +330,11 @@ services:
|
||||
- "9034:8083" # gRPC streaming
|
||||
volumes:
|
||||
- ./site-b-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro
|
||||
- ./site-b-node-b/data:/app/data
|
||||
# Same host directory, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is
|
||||
- site-b-node-b-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this mount is
|
||||
# not touched by an image rebuild, but the guard is lexical and cannot know that.
|
||||
- ./site-b-node-b/data:/data
|
||||
- site-b-node-b-data:/data
|
||||
- ./site-b-node-b/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -364,11 +364,11 @@ services:
|
||||
- "9043:8083" # gRPC streaming
|
||||
volumes:
|
||||
- ./site-c-node-a/appsettings.Site.json:/app/appsettings.Site.json:ro
|
||||
- ./site-c-node-a/data:/app/data
|
||||
# Same host directory, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is
|
||||
- site-c-node-a-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this mount is
|
||||
# not touched by an image rebuild, but the guard is lexical and cannot know that.
|
||||
- ./site-c-node-a/data:/data
|
||||
- site-c-node-a-data:/data
|
||||
- ./site-c-node-a/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -398,11 +398,11 @@ services:
|
||||
- "9044:8083" # gRPC streaming
|
||||
volumes:
|
||||
- ./site-c-node-b/appsettings.Site.json:/app/appsettings.Site.json:ro
|
||||
- ./site-c-node-b/data:/app/data
|
||||
# Same host directory, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this bind mount is
|
||||
- site-c-node-b-data:/app/data
|
||||
# Same named volume, mounted a second time OUTSIDE the content root (/app).
|
||||
# Secrets 0.6.2 refuses a store path under the content root; this mount is
|
||||
# not touched by an image rebuild, but the guard is lexical and cannot know that.
|
||||
- ./site-c-node-b/data:/data
|
||||
- site-c-node-b-data:/data
|
||||
- ./site-c-node-b/logs:/app/logs
|
||||
networks:
|
||||
- scadabridge-net
|
||||
@@ -424,3 +424,29 @@ services:
|
||||
networks:
|
||||
scadabridge-net:
|
||||
external: true
|
||||
|
||||
# Named volumes for every node's SQLite data dir (register row 79). On macOS/OrbStack a
|
||||
# bind mount crosses virtiofs, which does not give SQLite true fsync durability — both
|
||||
# site-a LocalDb files corrupted through that layer (row 69). Named volumes live on
|
||||
# native ext4 inside the OrbStack VM, taking virtiofs out of the write path entirely.
|
||||
# Host-side inspection still works read-only via ~/OrbStack/docker/volumes/<name>;
|
||||
# `docker compose down` preserves them; remove with `docker volume rm <name>`.
|
||||
# The appsettings/logs bind mounts stay: config is ro, and Serilog's append-only files
|
||||
# have no fsync-durability requirement worth losing `tail -f` from the repo tree for.
|
||||
volumes:
|
||||
central-node-a-data:
|
||||
name: scadabridge-central-node-a-data
|
||||
central-node-b-data:
|
||||
name: scadabridge-central-node-b-data
|
||||
site-a-node-a-data:
|
||||
name: scadabridge-site-a-node-a-data
|
||||
site-a-node-b-data:
|
||||
name: scadabridge-site-a-node-b-data
|
||||
site-b-node-a-data:
|
||||
name: scadabridge-site-b-node-a-data
|
||||
site-b-node-b-data:
|
||||
name: scadabridge-site-b-node-b-data
|
||||
site-c-node-a-data:
|
||||
name: scadabridge-site-c-node-a-data
|
||||
site-c-node-b-data:
|
||||
name: scadabridge-site-c-node-b-data
|
||||
|
||||
+2
-2
@@ -10,8 +10,8 @@ docker compose -f "$SCRIPT_DIR/docker-compose.yml" down
|
||||
|
||||
echo ""
|
||||
echo "Teardown complete."
|
||||
echo "Site data (SQLite DBs) and logs are preserved in node directories."
|
||||
echo "Node data (SQLite DBs) is preserved in named docker volumes; logs in node directories."
|
||||
echo ""
|
||||
echo "To also remove persistent data:"
|
||||
echo " rm -rf $SCRIPT_DIR/site-a-node-a/data $SCRIPT_DIR/site-a-node-b/data"
|
||||
echo " docker volume rm \$(docker volume ls -q --filter name=scadabridge-central-node --filter name=scadabridge-site-)"
|
||||
echo " rm -rf $SCRIPT_DIR/*/logs"
|
||||
|
||||
@@ -359,18 +359,25 @@ database from its peer rather than letting it rejoin.
|
||||
|
||||
### Docker-rig data files on macOS/OrbStack — a virtiofs corruption + availability hazard
|
||||
|
||||
**Status: MITIGATED 2026-08-18 (register row 79) — the rigs' data dirs moved to named docker
|
||||
volumes**, taking virtiofs out of the SQLite write path entirely. The hazard description and the
|
||||
proven recovery runbook below are retained because they explain *why* the volumes exist, and
|
||||
because failure mode 2 (the VM idle-sleep) is unaffected by the move.
|
||||
|
||||
**Scope: the local `docker/` and `docker-env2/` development rigs on macOS only.** This is a
|
||||
host-level property of the filesystem the rig's data files sit on, **not a ScadaBridge defect** and
|
||||
not something a production Linux deployment is exposed to. It is documented here because it has
|
||||
already cost two databases and half a session, and because the recovery is non-obvious.
|
||||
|
||||
Every ScadaBridge data file on the rig rides a **relative bind mount** out of the repo working tree
|
||||
— `./site-a-node-a/data:/app/data` and its siblings, which is where `site-localdb.db`,
|
||||
`auditlog.db` and the secrets store all live. On macOS that path crosses **virtiofs** to reach the
|
||||
Linux VM, and **virtiofs does not give SQLite true `fsync` durability**. SQLite's crash-safety
|
||||
contract assumes a successful `fsync` means the bytes are on stable storage; when that is not true,
|
||||
an abrupt container kill mid-write can leave a torn file. `docker compose up --force-recreate`
|
||||
— which `deploy.sh` uses — is exactly such an abrupt kill.
|
||||
Until 2026-08-18, every ScadaBridge data file on the rig rode a **relative bind mount** out of the
|
||||
repo working tree — `./site-a-node-a/data:/app/data` and its siblings, which is where
|
||||
`site-localdb.db`, `auditlog.db` and the secrets store all live. On macOS that path crosses
|
||||
**virtiofs** to reach the Linux VM, and **virtiofs does not give SQLite true `fsync` durability**.
|
||||
SQLite's crash-safety contract assumes a successful `fsync` means the bytes are on stable storage;
|
||||
when that is not true, an abrupt container kill mid-write can leave a torn file.
|
||||
`docker compose up --force-recreate` — which `deploy.sh` uses — is exactly such an abrupt kill.
|
||||
Today the same paths are **named volumes** (`scadabridge-<node>-data`,
|
||||
`scadabridge-env2-<node>-data`) on native ext4 inside the OrbStack VM, where `fsync` is real.
|
||||
|
||||
Two distinct failure modes have been observed live (both 2026-08-15):
|
||||
|
||||
@@ -421,13 +428,16 @@ be restored from a backup instead.
|
||||
1. **Stop BOTH nodes of the site pair, together.** Not one at a time — this is the same
|
||||
stop-and-start-together rule as *Site Pair Upgrades* above, for the same reason.
|
||||
2. **Quarantine the database and its sidecars — do not delete them.** Rename all three, keeping
|
||||
them for forensics:
|
||||
them for forensics. The data dir now lives in the node's named volume, so run the renames
|
||||
through a throwaway container (don't write through the host-side `~/OrbStack` view):
|
||||
|
||||
```sh
|
||||
# per affected node's data directory
|
||||
mv site-localdb.db site-localdb.db.corrupt-$(date +%F)
|
||||
mv site-localdb.db-wal site-localdb.db-wal.corrupt-$(date +%F)
|
||||
mv site-localdb.db-shm site-localdb.db-shm.corrupt-$(date +%F)
|
||||
# per affected node — volume name is scadabridge-<node>-data
|
||||
docker run --rm -v scadabridge-site-a-node-a-data:/d alpine sh -c '
|
||||
cd /d
|
||||
mv site-localdb.db site-localdb.db.corrupt-$(date +%F)
|
||||
mv site-localdb.db-wal site-localdb.db-wal.corrupt-$(date +%F)
|
||||
mv site-localdb.db-shm site-localdb.db-shm.corrupt-$(date +%F)'
|
||||
```
|
||||
|
||||
The `-wal` and `-shm` files **must** move with the main file. Leaving a stale WAL beside a fresh
|
||||
@@ -443,16 +453,25 @@ On the live recovery, both quarantined files were large (394 MB and 416 MB), the
|
||||
`connected: true` / backlog 0, and a schema change in flight at the time additionally proved the
|
||||
digest handshake green.
|
||||
|
||||
#### Mitigation options, with their trade-offs
|
||||
#### Mitigation applied (2026-08-18): named docker volumes
|
||||
|
||||
None of these is applied today; the rig deliberately still runs on bind mounts, and the accepted
|
||||
position is the runbook above. Recorded so the choice is a choice:
|
||||
The "accept + runbook" position was retired without waiting for a second corruption. All ten
|
||||
data dirs (8 `docker/` nodes + 2 `docker-env2/` site nodes) now ride named volumes
|
||||
(`scadabridge-<node>-data` / `scadabridge-env2-<node>-data`), declared in each compose file with
|
||||
explicit `name:`s and seeded from the old bind-mount dirs at migration time. What changed the
|
||||
original trade-off calculus: the assumed cost — losing host-side inspectability — turned out to
|
||||
be much smaller than documented, because OrbStack exposes every volume on the Mac at
|
||||
`~/OrbStack/docker/volumes/<name>/`, so `site-localdb.db` can still be opened from the
|
||||
host for **read-only inspection**. (Never write through that path: it reintroduces exactly the
|
||||
virtiofs layer the volumes exist to avoid.) The appsettings and logs bind mounts deliberately
|
||||
stayed: config is read-only, and Serilog's append-only log files have no fsync-durability
|
||||
requirement worth losing `tail -f` from the repo tree for.
|
||||
|
||||
| Option | Gains | Costs |
|
||||
|--------|-------|-------|
|
||||
| Move rig data dirs to **named docker volumes** | Files live on native ext4 inside the VM, out of virtiofs entirely — removes the corruption mechanism at the source | Loses the host-side inspectability the debugging sessions lean on constantly: no more opening `site-localdb.db` from the Mac, no `ls` of a node's data dir from the repo tree. Requires a compose change. |
|
||||
| **Disable OrbStack's idle pause** | Removes failure mode 2 (the mid-operation sleep and the virtiofs `refresh_nodeid` storm) | A host-level setting outside this repo, so it cannot be version-controlled or applied by `deploy.sh` — it silently does not exist on a new machine. Does nothing about failure mode 1. |
|
||||
| **Accept, and use this runbook** (current) | No changes to compose or host config; full host-side inspectability retained | Corruption recurs on the next unlucky `--force-recreate`, and can sit latent until a later upgrade turns it into a crash loop. |
|
||||
Residual exposure: **failure mode 2 is availability-only and still real** — the VM can still
|
||||
idle-sleep mid-operation and hang `docker` until `orb restart docker`; data now survives that
|
||||
on ext4. Disabling OrbStack's idle pause remains an un-version-controllable host setting and was
|
||||
left alone. The old `docker/<node>/data/` dirs were left in place as a dormant pre-migration
|
||||
backup; they are no longer mounted and can be deleted once the volumes have proven themselves.
|
||||
|
||||
### Central-Site Communication
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ All 7 fix-now items landed via PLAN-04/05/06/07/08 (verified in review 08 round
|
||||
| 76 | ~~**CLI README's `audit-config query` JSON sample omits `afterStateJson`/`bundleImportId`** *(Low, docs)*~~ **RESOLVED 2026-08-16 (`4ae57bdb`).** Sample now shows both fields with their nullability semantics, and was grown to two entries mirroring the table-output sample below it row-for-row — which is what makes the "the table drops these two fields" sentence checkable. | Row-57 fix (2026-08-15) | Closed 2026-08-16. | — |
|
||||
| 77 | ~~**Test pattern: reflecting `ActorCell.Actor` right after `ResolveOne` is unordered vs child construction** *(Low, test-infra)*~~ **RESOLVED 2026-08-16 — swept clean, no commit needed.** Full `tests/` sweep: `ActorCell` (1 hit — the already-fixed file's comment), `ActorRefWithCell` (1 — fixed file :135), `.Underlying` (1 — same line), `BindingFlags` near actor/cell (2 — same file), `.Cell` (0), `ResolveOne` (3 files: the fixed one; `SiteAlarmLiveCacheServiceTests:432` resolves-then-`Sys.Stop`s, benign; `ReconcileStartsInstanceTests:363` uses the ref only for `IActorRef` equality, benign), `UnderlyingActor` (6 files — all Akka.TestKit `TestActorRef<T>` via `ActorOfAsTestActorRef`, an ORDERED mechanism, not this pattern). The unordered idiom exists in exactly one place and already carries the bounded-deadline retry. | Row-40 session (2026-08-15) | Closed 2026-08-16. | — |
|
||||
| 78 | ~~**Configs arriving via LocalDb replication start no InstanceActor — row 67's gap through a side door** *(Medium-Low, availability; proven live 2026-08-15)*~~ **RESOLVED 2026-08-16 (`e606b24a`, with row 72 — the DM instance sweep).** A periodic sweep on the Deployment Manager **singleton** (`ScadaBridge:SiteRuntime:InstanceSweepIntervalSeconds`, default 60, validated `>= 0`, `0` disables — default is live everywhere with no rig config change, which is intended): each tick re-reads the durable rows off-thread and, on the actor thread, starts any **enabled** stored config with no running actor. Arrival-path-agnostic by design — covers replication seeding, post-boot snapshot resyncs, and any future path. A LocalDb change hook was considered and rejected on evidence: it covers only the replication path and only the replicated half of the fleet (site-b/c aren't), AND `ILocalDb` 0.3.0 exposes no change-notification API at all, so a hook means a cross-repo upstream capability to close a site-local gap. Warning site event on a real heal names the sweep as the trigger; silent when there is nothing to do. Fail-before (sweep unarmed): 5 of 10 new tests fail including the exact live symptom ("A replication-seeded config for an enabled instance left no running Instance Actor"); 5 negative guards (disabled / in-flight-deploy deferral / matching-revision no-op / silence / interval-0) green in both states. Design note: `docs/plans/2026-08-15-instance-sweep-self-heal-design.md`. **LIVE-PROVEN at close-out (2026-08-16) in the exact original failure sequence:** site-a pair stopped, node-a's LocalDb file quarantined, pair restarted together — node-a's 08:45:55 reconcile reported "0 fetched" (replication had seeded the fresh store; central rightly saw no gap — the precise side door), and at 08:46:50, one sweep interval later, BOTH enabled instances started with the Warning `Instance rig-probe-1 started by deployment-manager sweep self-heal — its config is present in the deployed-config store but no Instance Actor was running until now`; `debug snapshot` live with no extra restart. The probe also exposed the DCL sibling gap — row 85. | This wave's close-out (2026-08-15) | Closed 2026-08-16. | — |
|
||||
| 79 | **OrbStack virtiofs is a live corruption + availability hazard for the rig's bind-mounted SQLite files** *(Medium, rig infra — host-level, not product)* — **mitigation chosen 2026-08-16: accept + document (`851b528c`).** The full hazard description + the proven quarantine-and-reseed recovery runbook now live in `docs/deployment/topology-guide.md` (new section), including the `orb restart docker` VM-sleep recovery and the named-volumes/idle-pause alternatives with their trade-offs. Row stays OPEN as the hazard tracker: a recurrence should trigger actually moving to named volumes or disabling idle-pause rather than re-running the runbook a third time. Original evidence: the OrbStack VM idle-SLEPT mid-session (`vmgr … msg=sleep`) leaving docker hung ~20 min until `orb restart docker`, with virtiofs `refresh_nodeid: race with new lookup` ERRORs on wake; both site-a LocalDb files (row 69) corrupted through this bind-mount layer — macOS virtiofs does not give SQLite true fsync durability, and abrupt container kills mid-write compound it. | This wave's close-out (2026-08-15); runbook shipped 2026-08-16 | Runbook shipped; hazard accepted for now | **Next corruption occurrence ⇒ stop accepting: move the rig data dirs to named volumes or disable OrbStack idle-pause** |
|
||||
| 79 | ~~**OrbStack virtiofs is a live corruption + availability hazard for the rig's bind-mounted SQLite files**~~ **RESOLVED 2026-08-18 — the prescribed remedy executed without waiting for a second corruption: all ten rig data dirs (8 `docker/` nodes + 2 `docker-env2/` site nodes) moved to named docker volumes** (`scadabridge-<node>-data` / `scadabridge-env2-<node>-data`, explicit `name:`s in both compose files), seeded from the old bind dirs with both rigs stopped, then redeployed and live-verified: site-a `localDbReplicationConnected: true` / backlog 0 over the migrated files, deployed instances resubscribed on boot (data continuity proven), env2 stable with 0 restarts. The trade-off that had deferred this — losing host-side inspectability — turned out to be mostly moot: OrbStack exposes every volume on the Mac at `~/OrbStack/docker/volumes/<name>/` (read-only inspection; never write through it). The rebuild also surfaced latent env2 drift: env2 was running a pre-Secrets-0.6.2 image, so on the fresh image all four env2 app nodes failed the new store-path validation at boot — fixed by porting the main rig's `x-secrets-store-env` posture (`Secrets__SqlitePath=/data/…` + `/data` second mounts, incl. NEW env2 central data volumes; env2 centrals previously kept data on the ephemeral overlay). Residual accepted: failure mode 2 (VM idle-sleep, `orb restart docker` recovery) is availability-only and unaffected — data now survives it on ext4. Hazard history + recovery runbook retained in `docs/deployment/topology-guide.md` (section updated). Old `docker/<node>/data/` dirs left in place as dormant pre-migration backups. | This wave's close-out (2026-08-15); runbook 2026-08-16; migration executed 2026-08-18 | Closed 2026-08-18. | — |
|
||||
| 80 | ~~**Unreplicated sites report NON-NULL `LocalDbReplicationConnected: false` / `LocalDbOplogBacklog: 0` — contradicting the documented nullable semantics** *(Low-Medium, observability correctness)*~~ **RESOLVED 2026-08-16 (`a2f0086f`, with row 73).** Root cause was type-level: `ISiteHealthCollector.SetLocalDbReplicationStatus(bool …)` made null INEXPRESSIBLE for the connected flag, and the Host wired the bridge unconditionally — the replication engine is registered on every site node and idles answering `false`/`0` with no peer. Fix mirrors the `SiteEventLogReplicationCheck` precedent exactly: the Host evaluates `SiteLocalDbSetup.ReplicationIsConfigured` once and passes it as a third provider to `AddLocalDbReplicationHealthBridge` (the SAME captured value now also feeds `SiteEventLogReplicationCheck`, so the two surfaces can't disagree); unconfigured short-circuits to `(null, null)`; the collector widens to `bool?`. The rest of the chain needed NO change — proto already `BoolValue`/`Int64Value` wrappers, `NextBacklogAboveSince` already leaves the anchor unchanged on null, KPI source already conditional: row 33's machinery was correct and had simply never received a null (its null branch was dead code in production until now). `false` now means exactly one thing: configured, peer session down = real fault. Trap that produced the bug, now pinned in tests: configured-ness is `PeerAddress` **OR** `ApiKey` — "no PeerAddress" is NOT "unreplicated" (a passive-half node carries the key alone); the pre-existing Host fixture asserting `false`/`0` was exactly such a node and its assertion is CORRECT — kept with an explaining comment, plus a new genuinely-unreplicated fixture. Fail-before: gate behaviourally reverted → `UnreplicatedSite_HealthReport_ReportsNullForBothLocalDbFields` fails `Expected: null / Actual: False`. | Found at the previous wave's close-out verifying row 33 | Closed 2026-08-16. | — |
|
||||
| 81 | ~~**The site-event-log storage cap cannot be enforced on a replicated node with an undrained oplog** *(Medium-Low, measured 2026-08-16 during the row-34 work)*~~ **RESOLVED 2026-08-17 (`ddb3c32e`) — but NOT via the row's proposed `dbstat` fix, which is impossible on this stack.** Probed BEFORE implementing: the bundled SQLite 3.53.3 has no `ENABLE_DBSTAT_VTAB` (`pragma_module_list` returns 10 modules, none of them `dbstat`/`sqlite_dbpage`); pinned by `SqliteCapabilityTests`, written to FAIL if a future bump enables it (row 94). Switching SQLite builds rejected — the provider is shared with `ZB.MOM.WW.LocalDb` fleet-wide. The cap instead measures `site_events` + its four indexes via a sampled page-layout model: row width sampled 500 rows from each end per tick (`octet_length`), whole-page table cost (linear models under-estimate near-half-page rows by ~40%), clamped by whole-file logical size (a true upper bound). Index pages count against the cap (~34% of footprint — the 1 GB default was sized against a figure that included them). Measured model error −0.8%/+8.4%/+12.8%, biased high on purpose; cross-checks row 34's independent 393 B/row at 397.2. Guard semantics re-based: (1) unsampleable row width ⇒ refuse to trim (checked BEFORE the cap comparison); (2) the old "size did not decrease" guard now runs on the table-scoped estimate, so replication bookkeeping can no longer trip it — it now means exactly one thing: events arriving as fast as the trim removes them. Bytes-per-row held fixed per tick (re-sampling per batch would jitter the estimate through guard 2 — the same self-defeating measurement as the row itself). LocalDb bookkeeping explicitly NOT managed here (`MaxOplogRows`/tombstone retention own it); topology guide gained a volume-sizing subsection stating the honest budget. Fail-before: reverting to whole-file semantics fails 5 new tests and reproduces the exact original signature (rows destroyed, cap still unmet). SiteEventLogging 104/104, Host 520/520. No config keys added or renamed. Decision memo: `docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md`. Model limitations recorded as rows 92/93. | Row-34 measurement harness; §5 of the measurement doc (postscript appended) | Closed 2026-08-17. | — |
|
||||
| 82 | ~~**`AuditLog.Tests` flakes under parallel full-solution runs — `MsSqlMigrationFixture` contention** *(Low, test-infra)*~~ **RESOLVED 2026-08-17 (`da23f2dc`) — three mechanisms, and the row's "per-run database name" prescription was already in place (the fixture has minted `ScadaBridgeAuditMigTest_<guid>` per class since it was written).** (1) The headline fingerprint was create+migrate SATURATION: nothing bounded how many fixtures ran `CREATE DATABASE` + the ~100-migration chain at once (measured peak: 45 concurrent test DBs / 19 blocked requests / 98 sessions on one container), pushing DDL past EF's default 30 s command timeout — every observed failure was `..ctor → MigrateAsync → SqlException 258 Execution Timeout Expired`, which xunit reports against every test in the class (hence "29 failures"). Fix: a machine-wide counting semaphore on `sp_getapplock` in `master` (`ScadaBridgeTestMigrate/0..N-1`, 4 slots, `SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS` override) held across create+migrate — the arbiter must live on the shared SERVER because the contention is cross-process; gate connection is deliberately UNPOOLED (a pooled one defers the session-lock release until reuse, stranding a slot); crashed process ⇒ SQL releases the lock; a gate deadline miss proceeds UNGATED so the throttle can never itself be a red. Timeouts raised to 300 s migrate / 120 s create-drop (bounded queue = busy but progressing); once-per-process sweep of orphaned `ScadaBridgeAuditMigTest_*` >2 h; connectivity probe retries 3× (a busy server misread as absent flips every MSSQL test to a green-looking Skip — worse than a red). (2)+(3) Two more flakes behind it: `PartitionPurgeTests` and `NotifyDispatcherAuditTrailTests` LEAKED 100 ms-tick actors (purge / outbox dispatcher) whose only stop was timeout-bounded `TestKit.Dispose` — a leaked dispatcher even delivered the NEXT test's notification through its own exhausted adapter, stranding it `Retrying`. Fixed with `Watch`/`Stop`/`ExpectTerminated` in the creating test. Evidence: 10× concurrent ConfigurationDatabase.Tests went from 10/10 processes failing (17–77 failures each) to 3 consecutive 10/10-green runs; the AuditLog 3×+1× mix from failing to 3 clean runs; standalone AuditLog 383/383, ConfigurationDatabase 395/395 (no runtime regression), SiteCallAudit 66/66 (inherits the fixture); zero orphan DBs or stuck app locks left. Host.Tests/PlaywrightTests hit the PERSISTENT DB on an already-serialized axis — untouched. Systemic residue recorded as rows 95/96. Doc: `docs/test_infra/test_infra_db.md`. | Found independently by two agents in the 2026-08-16 wave | Closed 2026-08-17. | — |
|
||||
@@ -84,9 +84,9 @@ All 7 fix-now items landed via PLAN-04/05/06/07/08 (verified in review 08 round
|
||||
| 89 | ~~**Connection-only artifact deploy does not reach the peer node's DCL**~~ **RESOLVED 2026-08-17 (merge `6cfb6a79`, agent commit `c57e74e6`) — premise corrected TWICE, then fixed.** (1) The standby's DCL is **EMPTY**, not "serving the old endpoint": `DeploymentManagerActor` (role-scoped singleton) is the sole sender of `CreateConnectionCommand` and only populates its own node's `dcl-manager`. The real defect was **create-time and not replication-specific**: `_createdConnections` is per singleton *incarnation*, and `data_connection_definitions` had NO production reader — so any restart/failover rebuilt every connection from the flattened-config snapshots and silently discarded newer connection-only artifact deploys (identical loss on an unreplicated single-node restart). (2) Arbitration decided and shipped: **the definitions store wins wherever it has an opinion; flattened config is authoritative only for names the store never owned** (deploy-only sites unchanged — memo §7.4's objection). A `_connectionDefinitions` overlay refreshes from `ReadStoreSnapshotAsync` at boot AND every sweep tick, applied before instance creation; ping-pong killed by construction (every ensure path resolves the *winner* before hashing, so one config per hash slot). Fail-before: 5/8 new `DclConnectionArbitrationTests` fail with the overlay inert. Decision memo: `docs/plans/2026-08-17-dcl-connection-arbitration-decision.md` (records the accepted residuals: re-adds land at next instance start, not instantly — mirrors the removal's own "bad quality until redeploy" semantic, pinned by test). The discovery that a *live* connection actor is never reconfigured in place on ANY node is a separate pre-existing defect → row 97. **Live rig probe at wave close-out (post-redeploy):** the singleton node logged exactly ONE boot-time `Created DCL connection OPC PLC Simulator` with zero update churn across later sweep ticks (no ping-pong from the overlay), the standby stayed silent (topology as corrected), all three rig-probe-1 tags Good, site-a replication connected / backlog 0. | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — |
|
||||
| 90 | ~~**No removal path for a connection deleted on the peer**~~ **RESOLVED 2026-08-17 (merge `6cfb6a79`, agent commit `c57e74e6`), with row 89.** Both removal paths now share `RemoveConnectionCentrally` + an in-memory tombstone that `EnsureDclConnections` honors — which also closed a pre-existing bug where an artifact removal was **undone by the very next instance start**. The sweep detects a previously-definition-sourced row's disappearance via its own overlay bookkeeping. Accepted residuals, documented in the decision memo: tombstones are not durable across a restart (by decision — matches the long-standing restart-resurrection semantic), and a delete is undetectable on a node that never adopted the row (the stronger "non-empty table ⇒ complete set" inference would stop live connections on a guess; deliberately not taken). | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — |
|
||||
| 91 | ~~**`SiteScriptCompileCacheTests.GetOrAdd_SameCodeAndGlobals_ComputesOnce` is parallel-load fragile**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `26e3a1b9`) — row-74 pattern applied (exemplar `98c94588`), and wider than the named test.** Two habits: a per-test-instance guid key token (no other class can collide with or serve this test's entries, making `factoryCalls`/`Assert.Same` exact and owned) + one-sided bounds on the process-global counters (`>= baseline + n`; safe because all `Clear()` callers share the serialized collection). Four assertions changed, two got STRONGER (`GetOrAdd_SameCode_DifferentGlobals` had a second absolute pair; `Overflow_StaysWithinTheBound` gained an eviction probe over its own keys — a slack bound is not proof a sweep ran; its `ForeignEntrySlack = 512` is a judgement call, the eviction probe is the load-bearing assertion). Sibling absolutes in the same class and in `DeploymentWarmThenGateTests` fixed too. SiteRuntime.Tests run twice at full parallelism, 633/633 both. The underlying process-global `Clear()` cross-talk hazard remains → row 100. | Found by the 2026-08-17 row-85 fix session | Closed 2026-08-17. | — |
|
||||
| 92 | **The event-log cap's size model is sample-based and blind to mid-history width shifts** *(Low, accuracy limitation — documented in memo §3.1)* | Row-81 fix session (2026-08-17) | Row width is sampled 500 rows from each END of `site_events`; a stream whose row width changes sharply mid-history is mis-sized until the trim reaches that region. Self-correcting per tick (each trim exposes new tail rows to the sample), and the model's deliberate +8–13% high bias absorbs modest shifts, but a pathological width profile could over- or under-trim for several ticks. | First field report of the cap trimming visibly wide of its target, or any rework of the size model (see rows 93/94) |
|
||||
| 93 | **The cap model's page-layout constants are empirical for the CURRENT `site_events` schema** *(Low, maintenance trap)* | Row-81 fix session (2026-08-17) | Fill factors (0.78/0.75), cell overheads (16/8) and especially `IndexedColumnCount = 4` are calibrated to today's table + four indexes. A schema change — new column, new index — silently invalidates them; a fifth index in particular would be silently under-counted. Any `site_events` schema migration must re-run the memo §4 calibration and update the constants together. | Any schema change to `site_events` or its indexes |
|
||||
| 94 | **`SqliteCapabilityTests` is a deliberate tripwire — do not "fix" it by weakening the assertion** *(Low, standing note)* | Row-81 fix session (2026-08-17) | The test FAILS BY DESIGN if a future SQLite bump ships `dbstat`/`sqlite_dbpage` support. The correct response at that point is to REPLACE the sampled page-layout model with exact `dbstat` measurement (retiring rows 92/93 with it), not to relax the test. The test's own comment says the same. | `SqliteCapabilityTests` fails after a SQLite/provider bump |
|
||||
| 92 | **The event-log cap's size model is sample-based and blind to mid-history width shifts** *(Low, accuracy limitation — documented in memo §3.1)* | Row-81 fix session (2026-08-17) | Row width is sampled 500 rows from each END of `site_events`; a stream whose row width changes sharply mid-history is mis-sized until the trim reaches that region. Self-correcting per tick (each trim exposes new tail rows to the sample), and the model's deliberate +8–13% high bias absorbs modest shifts, but a pathological width profile could over- or under-trim for several ticks. **Reviewed 2026-08-18: accepted by design** — inherent to any sampled model, not closable by code short of the row-94 `dbstat` replacement; the row-93 schema pin does not (and cannot) cover it. | First field report of the cap trimming visibly wide of its target, or the row-94 `dbstat` replacement (which retires this row with the model) |
|
||||
| 93 | ~~**The cap model's page-layout constants are empirical for the CURRENT `site_events` schema**~~ **RESOLVED 2026-08-18 — the silent trap is now loud.** `SiteEventsSchemaCalibrationTests` (SiteEventLogging.Tests) pins the exact shape the calibration assumed: the eight-column list the sampler's `octet_length` sum covers, the index set as exactly the four secondary indexes `IndexedColumnCount = 4` charges plus the PK autoindex (absorbed into the measured fill factors), and each index's single covered column. Any `site_events` schema migration — new column, fifth index, composite widening — now fails the suite with a message directing to the memo §4 recalibration and the constants + sampling-SQL update, instead of silently mis-sizing the cap. Suite green 110/110. | Row-81 fix session (2026-08-17); pinned by test 2026-08-18 | Closed 2026-08-18. | — |
|
||||
| 94 | **`SqliteCapabilityTests` is a deliberate tripwire — do not "fix" it by weakening the assertion** *(Low, standing note)* | Row-81 fix session (2026-08-17) | The test FAILS BY DESIGN if a future SQLite bump ships `dbstat`/`sqlite_dbpage` support. The correct response at that point is to REPLACE the sampled page-layout model with exact `dbstat` measurement (retiring row 92 — and the row-93 schema pin `SiteEventsSchemaCalibrationTests`, added 2026-08-18, goes with the model too), not to relax the test. The test's own comment says the same. | `SqliteCapabilityTests` fails after a SQLite/provider bump |
|
||||
| 95 | ~~**The leaked 100 ms-tick test-actor pattern is systemic — three more classes carry it**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `34a8079d`).** All three classes live in AuditLog.Tests (premise correction: none in SiteCallAudit.Tests). `Watch`/`Stop`/`ExpectTerminated` replicated from `da23f2dc` at every creation site: 11 in `AuditLogPurgeActorTests`, 10 in `SiteAuditReconciliationActorTests`, 2 in `OutageReconciliationTests`. Second premise correction: `OutageReconciliationTests` DID call `Sys.Stop` — but fire-and-forget AND from the last line of the `await using` block, so an in-flight tick could still be writing to MSSQL during read-back and any earlier assertion failure skipped the stop entirely; stops moved before the read-backs, plus the `AuditLogIngestActor` that test 2 was discarding outright (holding a disposed provider) is now stopped too. Known lesser sibling habit left in place: EventStream probes are never unsubscribed (bounded by the per-test ActorSystem — noted, not a row). AuditLog.Tests 383/383. | Row-82 fix session (2026-08-17) | Closed 2026-08-17. | — |
|
||||
| 96 | ~~**xunit parallelism is uncapped almost everywhere**~~ **RESOLVED 2026-08-17 (merge `106e467d`, agent commit `d2d4db49`) — policy made and shipped: per-suite caps for the resource-heavy suites, pure unit suites stay uncapped.** The four MSSQL-fixture users (`ConfigurationDatabase.Tests`, `AuditLog.Tests`, `SiteCallAudit.Tests`, `Host.Tests` — verified before capping) get `xunit.runner.json` with `maxParallelThreads: 4`, **deliberately matching row-82's `sp_getapplock` slot default** so the in-process and cross-process limits agree; no collections serialized. `IntegrationTests` (already fully serialized) and `CentralUI.PlaywrightTests` (already `maxParallelThreads: 1`, stricter than needed) left alone. Cap verified live (216% CPU uncapped vs 145% capped on ConfigurationDatabase.Tests); cost is nil — these suites are latency-bound on MSSQL/sockets (14–25% CPU), durations unchanged within noise. Policy documented in `docs/test_infra/test_infra.md` → *Test Parallelism Policy* (two-axis model, per-suite table, default for new projects), cross-linked from `test_infra_db.md`. Found-not-fixed: `StoreAndForward.Tests/xunit.runner.json` is a no-op (pins the default `parallelizeTestCollections: true`) — harmless, tidy on next touch. | Row-82 fix session (2026-08-17) | Closed 2026-08-17. | — |
|
||||
| 97 | ~~**A live DCL connection actor is never reconfigured in place — on ANY node**~~ **RESOLVED 2026-08-17 (merge `9dbda4a2`, agent commit `7c1268d8`) — the memo's §6 sketch shipped, plus two things the sketch missed.** `EnsureDclConnection` now has three outcomes off its one hash slot: unchanged → nothing; new → create; **changed → `ReconfigureConnectionCommand`** (arbitration/tombstones byte-for-byte untouched). The manager forwards to the owning actor (unknown name falls back to create); `DataConnectionActor` handles it in **all three** lifecycle states and never stashes it, via a new shared `SwapAdapter` helper now also used by both failover sites. The sketch's two gaps, found in implementation: **endpoint selection must be re-armed to Primary** (a connection sitting on its old *backup* would otherwise dial the new backup first), and the deliberate link drop must **suppress the unstable-disconnect accounting** (else the swap itself could trip a failover). Also a premise-sharpening: "reconfigure while mid-connect" is the *primary* case, not an edge — a wrong endpoint loops in `Connecting` forever and never reaches `Connected`. Fail-before: 5/7 new `DataConnectionReconfigureTests` + 3 SiteRuntime ensure-path tests fail on the reverted behavior. **Live rig probe (post-redeploy, round trip):** connection 1's config changed (operationTimeoutMs 15000→16000→back) fired exactly TWO in-place reconfigures on the active node — `Reconfiguring DataConnectionActor for OPC PLC Simulator in place` → `Reconnected successfully on Primary endpoint` in 45 ms → `Re-subscribing 2 tags` — with all rig-probe-1 tags Good after each, the standby silent (0 reconfigure lines, topology as designed), and the old dead-end "has a new configuration" Warning never appearing. Rig config restored to original. | Found by the 2026-08-17 rows-89/90 fix session; memo `docs/plans/2026-08-17-dcl-connection-arbitration-decision.md` (§9 postscript appended) | Closed 2026-08-17. | — |
|
||||
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteEventLogging.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the <c>site_events</c> schema to the shape the storage-cap size model was calibrated
|
||||
/// against (register row 93, <c>docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md</c> §4).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <see cref="EventLogPurgeService.EstimateSiteEventsSize"/> hardcodes this schema twice over:
|
||||
/// the sampling SQL sums <c>octet_length</c> over the exact column list, and
|
||||
/// <c>IndexedColumnCount = 4</c> plus the fill-factor/overhead constants are empirical for
|
||||
/// exactly these four secondary indexes. A schema migration that adds a column or an index
|
||||
/// invalidates both SILENTLY — the new column's bytes are never sampled, a fifth index is
|
||||
/// never charged — so this test turns that drift into a loud failure. The fix on failure is
|
||||
/// NEVER to just update the assertions: re-run the memo §4 calibration, update the
|
||||
/// <see cref="EventLogPurgeService"/> constants and sampling SQL together, and only then
|
||||
/// re-pin the schema here.
|
||||
/// </remarks>
|
||||
public class SiteEventsSchemaCalibrationTests
|
||||
{
|
||||
private const string RecalibrationNotice =
|
||||
"site_events schema drifted from the size-model calibration. Re-run the calibration in " +
|
||||
"docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md §4 and update " +
|
||||
"EventLogPurgeService's constants + sampling SQL together with this pin (register row 93).";
|
||||
|
||||
private static SqliteConnection OpenWithSchema()
|
||||
{
|
||||
var connection = new SqliteConnection("Data Source=:memory:");
|
||||
connection.Open();
|
||||
SiteEventLogSchema.Apply(connection);
|
||||
return connection;
|
||||
}
|
||||
|
||||
private static List<string> ReadStrings(SqliteConnection connection, string sql)
|
||||
{
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = sql;
|
||||
var values = new List<string>();
|
||||
using var reader = cmd.ExecuteReader();
|
||||
while (reader.Read())
|
||||
values.Add(reader.GetString(0));
|
||||
return values;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Columns_MatchTheSampledOctetLengthSum_Exactly()
|
||||
{
|
||||
// The sampler's payload_bytes expression sums octet_length over exactly these eight
|
||||
// columns, in this order. A ninth column would contribute real disk bytes that the
|
||||
// model never counts, under-sizing every estimate.
|
||||
using var connection = OpenWithSchema();
|
||||
|
||||
var columns = ReadStrings(connection, "SELECT name FROM pragma_table_info('site_events')");
|
||||
|
||||
Assert.True(
|
||||
columns.SequenceEqual(
|
||||
["id", "timestamp", "event_type", "severity", "instance_id", "source", "message", "details"]),
|
||||
$"{RecalibrationNotice} Actual columns: {string.Join(", ", columns)}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Indexes_AreExactlyTheFourTheModelCharges_PlusThePkAutoindex()
|
||||
{
|
||||
// IndexedColumnCount = 4 charges four secondary index entries per row; the TEXT
|
||||
// primary key's autoindex is absorbed into the empirically-measured fill factors.
|
||||
// A fifth secondary index would be silently under-counted.
|
||||
using var connection = OpenWithSchema();
|
||||
|
||||
var indexes = ReadStrings(
|
||||
connection, "SELECT name FROM pragma_index_list('site_events') ORDER BY name");
|
||||
|
||||
Assert.True(
|
||||
indexes.SequenceEqual(
|
||||
[
|
||||
"idx_events_instance",
|
||||
"idx_events_severity",
|
||||
"idx_events_timestamp",
|
||||
"idx_events_type",
|
||||
"sqlite_autoindex_site_events_1",
|
||||
]),
|
||||
$"{RecalibrationNotice} Actual indexes: {string.Join(", ", indexes)}");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("idx_events_timestamp", "timestamp")]
|
||||
[InlineData("idx_events_type", "event_type")]
|
||||
[InlineData("idx_events_instance", "instance_id")]
|
||||
[InlineData("idx_events_severity", "severity")]
|
||||
public void EachIndex_CoversExactlyTheColumnTheSamplerSums(string indexName, string expectedColumn)
|
||||
{
|
||||
// The sampler's indexed_bytes expression sums octet_length(timestamp) +
|
||||
// octet_length(event_type) + octet_length(severity) + octet_length(instance_id) —
|
||||
// one column per index. Widening an index to a composite (or re-pointing it) would
|
||||
// change the real entry width without changing what is sampled.
|
||||
using var connection = OpenWithSchema();
|
||||
|
||||
var columns = ReadStrings(
|
||||
connection, $"SELECT name FROM pragma_index_info('{indexName}')");
|
||||
|
||||
Assert.True(
|
||||
columns.SequenceEqual([expectedColumn]),
|
||||
$"{RecalibrationNotice} Index {indexName} covers: {string.Join(", ", columns)}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user