8652eab98e
The Phase 2 soak's "LocalDb fails under load" blocker is NOT a product defect. Root cause: host-side (macOS) sqlite3 reads of the live, bind-mounted WAL databases. POSIX advisory locks do not propagate across the virtiofs boundary, so the host reader believes it is the only connection, checkpoints on close and resets the WAL to 0 bytes under the container. The container's still-mapped WAL index then references frames that no longer exist and every subsequent statement fails SQLITE_IOERR_SHORT_READ (522) -> primary code 10, permanently until the process reopens the database. Reproduced on demand both on the rig (one sqlite3 SELECT reset a 4.6 MiB WAL and produced the first error one second later) and in a minimal python:3.12-alpine repro with no LocalDb or .NET involved. Refuted the converse: a freshly-reopened node sustained the full soak load 10+ minutes with zero errors. The original brief's isolation was confounded - both nodes had been poisoned by the same sampling pass, and a poisoned standby looks healthy only because it issues almost no statements. Corrections annotated in place. Hardening shipped alongside: - SqliteErrorCodes.Describe: log SQLite primary AND extended codes at the LocalDb-adjacent catch sites (the missing extended code is what made the original diagnosis so slow). - SiteAuditTelemetryActor: stop touching ActorContext across an await (NotSupportedException), with regression coverage. - infra/reseed.sh: apply the MSSQL init scripts explicitly. The official mssql/server image does not implement /docker-entrypoint-initdb.d, so a fresh volume hung the reseed forever; compose mounts annotated as informational. Deliberately NOT done: detect-and-reopen self-heal in SqliteLocalDb. It defends only against external interference, which is now prevented at the source, and same-kernel production readers see the locks correctly. Build 0 warnings; SiteAuditTelemetryActorTests 9/9, SiteEventLogging 70/70. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
156 lines
5.4 KiB
Bash
Executable File
156 lines
5.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Full reseed of the ScadaBridge test cluster.
|
|
#
|
|
# Tears down infra + app containers, drops the MSSQL volume, brings
|
|
# everything back, lets EF Core migrations create the schema, replays
|
|
# infra/mssql/seed-config.sql for templates/scripts/data-connections, and
|
|
# re-seeds sites via docker/seed-sites.sh.
|
|
#
|
|
# Usage:
|
|
# infra/reseed.sh Full reseed (default seed file)
|
|
# infra/reseed.sh --seed PATH Replay a different seed SQL
|
|
# infra/reseed.sh --skip-teardown Replay seed against running stack
|
|
#
|
|
# Prerequisites:
|
|
# - Docker / OrbStack running
|
|
# - Python 3 with pymssql (used by infra/tools/mssql_tool.py + dump_seed.py)
|
|
# - Built scadabridge:latest image (docker/build.sh — deploy.sh runs it)
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
SEED_FILE="$SCRIPT_DIR/mssql/seed-config.sql"
|
|
SKIP_TEARDOWN=false
|
|
MGMT_URL="http://localhost:9000"
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--seed)
|
|
SEED_FILE="$2"
|
|
shift 2
|
|
;;
|
|
--skip-teardown)
|
|
SKIP_TEARDOWN=true
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//'
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unknown option: $1" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ ! -f "$SEED_FILE" ]; then
|
|
echo "Seed file not found: $SEED_FILE" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "=== ScadaBridge Reseed ==="
|
|
echo "Seed file: $SEED_FILE"
|
|
echo ""
|
|
|
|
if ! $SKIP_TEARDOWN; then
|
|
echo "--- Stage 1/6: tear down application containers ---"
|
|
"$PROJECT_ROOT/docker/teardown.sh"
|
|
|
|
echo ""
|
|
echo "--- Stage 2/6: wipe site SQLite state ---"
|
|
shopt -s nullglob
|
|
for d in "$PROJECT_ROOT"/docker/site-*/data; do
|
|
rm -rf "$d"/*
|
|
echo " cleared $d"
|
|
done
|
|
shopt -u nullglob
|
|
|
|
echo ""
|
|
echo "--- Stage 3/6: tear down infra (drops MSSQL volume) ---"
|
|
(cd "$SCRIPT_DIR" && docker compose down -v)
|
|
|
|
echo ""
|
|
echo "--- Stage 4/6: bring infra back up ---"
|
|
(cd "$SCRIPT_DIR" && docker compose up -d)
|
|
|
|
echo " Waiting for MSSQL to accept connections..."
|
|
until docker exec scadabridge-mssql /opt/mssql-tools18/bin/sqlcmd \
|
|
-S localhost -U sa -P 'ScadaBridge_Dev1#' -C -Q "SELECT 1" >/dev/null 2>&1; do
|
|
sleep 2
|
|
done
|
|
echo " MSSQL ready."
|
|
|
|
# The official mcr.microsoft.com/mssql/server image does NOT implement
|
|
# /docker-entrypoint-initdb.d, so the compose-mounted init scripts never run
|
|
# on their own — waiting for them here hangs forever on a fresh volume.
|
|
# Apply them explicitly instead (all are idempotent).
|
|
echo " Applying MSSQL init scripts (the mssql/server image has no initdb hook)..."
|
|
for f in mssql/setup.sql mssql/machinedata_seed.sql mssql/setup-env2.sql; do
|
|
echo " $f"
|
|
docker exec -i scadabridge-mssql /opt/mssql-tools18/bin/sqlcmd \
|
|
-S localhost -U sa -P 'ScadaBridge_Dev1#' -C -b < "$SCRIPT_DIR/$f"
|
|
done
|
|
echo " ScadaBridgeConfig present."
|
|
|
|
echo ""
|
|
echo "--- Stage 5/6: deploy central + site nodes ---"
|
|
"$PROJECT_ROOT/docker/deploy.sh"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- Stage 6a/6: wait for central cluster /health/ready ---"
|
|
until curl -fs "$MGMT_URL/health/ready" >/dev/null 2>&1; do
|
|
sleep 2
|
|
done
|
|
echo " Central cluster ready (EF Core migrations applied)."
|
|
|
|
echo ""
|
|
echo "--- Stage 6b/6: seed sites (CLI) ---"
|
|
# Sites must exist before the design seed: DataConnections.SiteId FKs to Sites.
|
|
"$PROJECT_ROOT/docker/seed-sites.sh"
|
|
|
|
echo ""
|
|
echo "--- Stage 6c/6: replay seed SQL ---"
|
|
docker exec -i scadabridge-mssql /opt/mssql-tools18/bin/sqlcmd \
|
|
-S localhost -U sa -P 'ScadaBridge_Dev1#' -C -d ScadaBridgeConfig -b < "$SEED_FILE"
|
|
echo " Seed replayed."
|
|
|
|
echo ""
|
|
echo "--- Stage 6d/6: restore encrypted secret config (CLI) ---"
|
|
# Configuration that lives in encrypted secret columns cannot be replayed from
|
|
# raw SQL: ASP.NET Data Protection ciphertext is non-deterministic and bound to
|
|
# the source key ring. Create/restore it through the app so the EF value
|
|
# converter encrypts against this cluster's key ring.
|
|
CLI="dotnet run --project $PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI --"
|
|
AUTH="--username multi-role --password password"
|
|
|
|
# ExternalSystemDefinitions Id 1 ("Test REST API") is inserted by the seed with
|
|
# a fixed identity but a NULL AuthConfiguration; set the API key here.
|
|
$CLI --url "$MGMT_URL" $AUTH external-system update \
|
|
--id 1 \
|
|
--name "Test REST API" \
|
|
--endpoint-url "http://scadabridge-restapi:5200" \
|
|
--auth-type ApiKey \
|
|
--auth-config "scadabridge-test-key-1"
|
|
echo " External-system auth config restored (encrypted)."
|
|
|
|
# The "Machine Data DB" database connection is referenced by name from the
|
|
# seeded TestDatabaseQuery script. It is not in seed-config.sql (its
|
|
# ConnectionString is an encrypted secret column); create it through the app.
|
|
$CLI --url "$MGMT_URL" $AUTH db-connection create \
|
|
--name "Machine Data DB" \
|
|
--connection-string "Server=scadabridge-mssql,1433;Database=ScadaBridgeMachineData;User Id=scadabridge_app;Password=ScadaBridge_Dev1#;TrustServerCertificate=true" \
|
|
|| echo " (Machine Data DB connection may already exist)"
|
|
echo " Database connection created (encrypted)."
|
|
|
|
echo ""
|
|
echo "=== Reseed complete ==="
|
|
echo ""
|
|
echo "Verify:"
|
|
echo " $PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net*/ZB.MOM.WW.ScadaBridge.CLI --url $MGMT_URL --username multi-role --password password template list"
|
|
echo ""
|
|
echo "To refresh the seed file from the current DB state:"
|
|
echo " python3 $SCRIPT_DIR/tools/dump_seed.py --output $SEED_FILE"
|