diff --git a/docker-env2/deploy.sh b/docker-env2/deploy.sh index 4975dc0d..25397dd7 100755 --- a/docker-env2/deploy.sh +++ b/docker-env2/deploy.sh @@ -13,7 +13,9 @@ fi echo "=== ScadaBridge Env2 Docker Deploy ===" -# Reuse the primary build (same scadabridge:latest image, same network creation) +# Reuse the primary build (same scadabridge:latest image, same network creation). +# SCADABRIDGE_IMAGE= builds and runs env2 on its own tag instead (build.sh and +# the compose file both read it), leaving scadabridge:latest untouched. "$SCRIPT_DIR/../docker/build.sh" # Ensure env2 databases exist on the shared scadabridge-mssql diff --git a/docker-env2/docker-compose.leader-auto-down.yml b/docker-env2/docker-compose.leader-auto-down.yml new file mode 100644 index 00000000..ea682e62 --- /dev/null +++ b/docker-env2/docker-compose.leader-auto-down.yml @@ -0,0 +1,28 @@ +# Opt-in overlay: run env2's central pair and its site-x pair on the in-repo +# `leader-auto-down` downing provider (LeaderAutoDowningProvider, ScadaBridge#106) +# instead of the shipped default `auto-down`. env2 is the soak environment for the +# #116 rollout; mirrors docker/docker-compose.leader-auto-down.yml. +# +# The shipped default is NOT changed by this file: the mounted appsettings still say +# `auto-down`; this overlay layers an environment override on top (env wins over JSON). +# +# A downing-provider change must reach both nodes of a pair at the same time - never +# roll it one node at a time. Apply (from the main checkout, both pairs together): +# +# docker compose -f docker-env2/docker-compose.yml -f docker-env2/docker-compose.leader-auto-down.yml \ +# up -d --force-recreate --no-deps central-a central-b site-x-a site-x-b +# +# Revert to the default by re-running the same command without the overlay file. +services: + central-a: + environment: + ScadaBridge__Cluster__SplitBrainResolverStrategy: "leader-auto-down" + central-b: + environment: + ScadaBridge__Cluster__SplitBrainResolverStrategy: "leader-auto-down" + site-x-a: + environment: + ScadaBridge__Cluster__SplitBrainResolverStrategy: "leader-auto-down" + site-x-b: + environment: + ScadaBridge__Cluster__SplitBrainResolverStrategy: "leader-auto-down" diff --git a/docker-env2/docker-compose.yml b/docker-env2/docker-compose.yml index 3157dd08..d8bcf39d 100644 --- a/docker-env2/docker-compose.yml +++ b/docker-env2/docker-compose.yml @@ -8,9 +8,13 @@ x-secrets-store-env: &secrets-store-env Secrets__SqlitePath: "/data/scadabridge-secrets.db" +# Node image: ${SCADABRIDGE_IMAGE:-scadabridge:latest}. Unset = the primary rig's shared +# image (the historic behaviour). Set SCADABRIDGE_IMAGE (same variable docker/build.sh +# tags with) to run env2 on its own build, e.g. SCADABRIDGE_IMAGE=scadabridge:env2-116, +# so an env2 rebuild never retags the image the primary rig's containers run. services: central-a: - image: scadabridge:latest + image: ${SCADABRIDGE_IMAGE:-scadabridge:latest} # An init process (tini) as PID 1, so a crashed dotnet process actually dies # instead of wedging the container at 100% CPU with the restart policy never # firing — mirrors docker/docker-compose.yml's init: true (ScadaBridge#34). @@ -75,7 +79,7 @@ services: restart: unless-stopped central-b: - image: scadabridge:latest + image: ${SCADABRIDGE_IMAGE:-scadabridge:latest} init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed @@ -137,7 +141,7 @@ services: restart: unless-stopped site-x-a: - image: scadabridge:latest + image: ${SCADABRIDGE_IMAGE:-scadabridge:latest} init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed @@ -166,7 +170,7 @@ services: restart: unless-stopped site-x-b: - image: scadabridge:latest + image: ${SCADABRIDGE_IMAGE:-scadabridge:latest} init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a # CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting + # actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed diff --git a/docker/build.sh b/docker/build.sh index 377e85e3..a8e7cd77 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -34,15 +34,21 @@ if [ -z "$SOURCE_REVISION" ] && git -C "$REPO_ROOT" rev-parse --git-dir >/dev/nu fi echo "Source revision: ${SOURCE_REVISION:-}" +# Image tag. Default scadabridge:latest (what docker/docker-compose.yml runs). Set +# SCADABRIDGE_IMAGE to build under another tag without touching :latest, e.g. so +# docker-env2 can run a different build while another agent uses the primary rig +# (ScadaBridge#116 env2 rollout). docker-env2/docker-compose.yml reads the same variable. +IMAGE="${SCADABRIDGE_IMAGE:-scadabridge:latest}" + # Build from repo root (so COPY paths in Dockerfile resolve correctly) -echo "Building scadabridge:latest image..." +echo "Building ${IMAGE} image..." # Note: ${NUGET_ARGS[@]+...} guards against the "unbound variable" error that # set -u raises on bash 3.2 (macOS default) when expanding an empty array. docker build \ - -t scadabridge:latest \ + -t "${IMAGE}" \ -f "$SCRIPT_DIR/Dockerfile" \ ${NUGET_ARGS[@]+"${NUGET_ARGS[@]}"} \ --build-arg "SOURCE_REVISION=${SOURCE_REVISION}" \ "$REPO_ROOT" -echo "Build complete: scadabridge:latest" +echo "Build complete: ${IMAGE}" diff --git a/docker/failover-drill.sh b/docker/failover-drill.sh index 3f723dbc..7a614ff1 100755 --- a/docker/failover-drill.sh +++ b/docker/failover-drill.sh @@ -20,7 +20,12 @@ # singletons, and /health/active goes 200 on the survivor WHILE THE VICTIM # IS STILL DOWN. Budget ~25s + singleton hand-over + health-probe margin. # -# DRILL_PAIR=central (default) | site-a | site-b | site-c — which 2-node pair. +# DRILL_ENV=primary (default) | env2 — which docker stack. env2 is the second +# stack in docker-env2/ (containers scadabridge-env2-*, Traefik :9100, single +# site pair site-x); TRAEFIK_URL defaults to its LB accordingly. +# +# DRILL_PAIR=central (default) | site-a | site-b | site-c (primary) | site-x (env2) +# — which 2-node pair. # Central health is on container port 5000 (also published as 9001/9002 and # routed by Traefik on 9000); a site's /health/* is on container port 8084 and # is NOT published, so the script probes every pair from a throwaway sidecar @@ -37,7 +42,13 @@ # fresh incarnation (standby), then that exactly one node of the pair is active. set -euo pipefail -TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}" +DRILL_ENV="${DRILL_ENV:-primary}" +case "$DRILL_ENV" in + primary) NAME_PREFIX="scadabridge-"; DEFAULT_TRAEFIK="http://localhost:9000"; SITE_PAIRS="site-a|site-b|site-c" ;; + env2) NAME_PREFIX="scadabridge-env2-"; DEFAULT_TRAEFIK="http://localhost:9100"; SITE_PAIRS="site-x" ;; + *) echo "ERROR: DRILL_ENV must be 'primary' or 'env2' (was '$DRILL_ENV')" >&2; exit 2 ;; +esac +TRAEFIK_URL="${TRAEFIK_URL:-$DEFAULT_TRAEFIK}" TIMEOUT_S="${TIMEOUT_S:-90}" REJOIN_TIMEOUT_S="${REJOIN_TIMEOUT_S:-180}" DRILL_MODE="${DRILL_MODE:-standby}" @@ -49,18 +60,17 @@ case "$DRILL_MODE" in standby|active) ;; *) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;; esac -case "$DRILL_PAIR" in - central) HEALTH_PORT=5000 ;; - site-a|site-b|site-c) HEALTH_PORT=8084 ;; - *) echo "ERROR: DRILL_PAIR must be central|site-a|site-b|site-c (was '$DRILL_PAIR')" >&2; exit 2 ;; -esac +if [ "$DRILL_PAIR" = central ]; then HEALTH_PORT=5000 +elif [[ "$DRILL_PAIR" =~ ^(${SITE_PAIRS})$ ]]; then HEALTH_PORT=8084 +else echo "ERROR: DRILL_PAIR must be central|${SITE_PAIRS} for DRILL_ENV=${DRILL_ENV} (was '$DRILL_PAIR')" >&2; exit 2 +fi case "$DRILL_ACTION" in kill|pause) ;; *) echo "ERROR: DRILL_ACTION must be 'kill' or 'pause' (was '$DRILL_ACTION')" >&2; exit 2 ;; esac -NODE_A="scadabridge-${DRILL_PAIR}-a" -NODE_B="scadabridge-${DRILL_PAIR}-b" +NODE_A="${NAME_PREFIX}${DRILL_PAIR}-a" +NODE_B="${NAME_PREFIX}${DRILL_PAIR}-b" PROBE="scadabridge-drill-probe-$$" docker run -d --rm --name "$PROBE" --network scadabridge-net "$PROBE_IMAGE" sleep 3600 > /dev/null @@ -96,7 +106,7 @@ else VICTIM_SEL="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE") fi -echo "pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} active=${ACTIVE} victim=${VICTIM_SEL} survivor=${SURVIVOR}" +echo "env=${DRILL_ENV} pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} active=${ACTIVE} victim=${VICTIM_SEL} survivor=${SURVIVOR}" HIT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ) VICTIM="$VICTIM_SEL" docker "$DRILL_ACTION" "${VICTIM}" > /dev/null @@ -202,5 +212,5 @@ echo "Exactly one active node in 10/10 samples." echo "Survivor downing/singleton evidence (last 20 matching log lines from ${SURVIVOR}):" docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Ei "leader-auto-down|auto-downing|-> Oldest|downing|marking.*down|removed" | tail -20 || true -echo "SUMMARY pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} down_logged_s=${DOWN_S:-n/a} rejoin_s=${ELAPSED}" +echo "SUMMARY env=${DRILL_ENV} pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} down_logged_s=${DOWN_S:-n/a} rejoin_s=${ELAPSED}" echo "Drill complete. Verify on the Health dashboard / Overview that both nodes show Up and exactly one is Primary."