chore(docker): env2 runs its own image tag, leader-auto-down overlay, drill targets env2 (#116)
- docker/build.sh: SCADABRIDGE_IMAGE picks the tag (default scadabridge:latest).
- docker-env2/docker-compose.yml: node image is ${SCADABRIDGE_IMAGE:-scadabridge:latest},
so env2 can run its own build without retagging the primary rig's image.
- docker-env2/docker-compose.leader-auto-down.yml: opt-in overlay for both env2 pairs,
mirroring docker/docker-compose.leader-auto-down.yml. Shipped default stays auto-down.
- docker/failover-drill.sh: DRILL_ENV=primary|env2 (container prefix, Traefik :9100,
pair site-x).
Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
@@ -13,7 +13,9 @@ fi
|
||||
|
||||
echo "=== ScadaBridge Env2 Docker Deploy ==="
|
||||
|
||||
# Reuse the primary build (same scadabridge:latest image, same network creation)
|
||||
# Reuse the primary build (same scadabridge:latest image, same network creation).
|
||||
# SCADABRIDGE_IMAGE=<repo:tag> builds and runs env2 on its own tag instead (build.sh and
|
||||
# the compose file both read it), leaving scadabridge:latest untouched.
|
||||
"$SCRIPT_DIR/../docker/build.sh"
|
||||
|
||||
# Ensure env2 databases exist on the shared scadabridge-mssql
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# Opt-in overlay: run env2's central pair and its site-x pair on the in-repo
|
||||
# `leader-auto-down` downing provider (LeaderAutoDowningProvider, ScadaBridge#106)
|
||||
# instead of the shipped default `auto-down`. env2 is the soak environment for the
|
||||
# #116 rollout; mirrors docker/docker-compose.leader-auto-down.yml.
|
||||
#
|
||||
# The shipped default is NOT changed by this file: the mounted appsettings still say
|
||||
# `auto-down`; this overlay layers an environment override on top (env wins over JSON).
|
||||
#
|
||||
# A downing-provider change must reach both nodes of a pair at the same time - never
|
||||
# roll it one node at a time. Apply (from the main checkout, both pairs together):
|
||||
#
|
||||
# docker compose -f docker-env2/docker-compose.yml -f docker-env2/docker-compose.leader-auto-down.yml \
|
||||
# up -d --force-recreate --no-deps central-a central-b site-x-a site-x-b
|
||||
#
|
||||
# Revert to the default by re-running the same command without the overlay file.
|
||||
services:
|
||||
central-a:
|
||||
environment:
|
||||
ScadaBridge__Cluster__SplitBrainResolverStrategy: "leader-auto-down"
|
||||
central-b:
|
||||
environment:
|
||||
ScadaBridge__Cluster__SplitBrainResolverStrategy: "leader-auto-down"
|
||||
site-x-a:
|
||||
environment:
|
||||
ScadaBridge__Cluster__SplitBrainResolverStrategy: "leader-auto-down"
|
||||
site-x-b:
|
||||
environment:
|
||||
ScadaBridge__Cluster__SplitBrainResolverStrategy: "leader-auto-down"
|
||||
@@ -8,9 +8,13 @@
|
||||
x-secrets-store-env: &secrets-store-env
|
||||
Secrets__SqlitePath: "/data/scadabridge-secrets.db"
|
||||
|
||||
# Node image: ${SCADABRIDGE_IMAGE:-scadabridge:latest}. Unset = the primary rig's shared
|
||||
# image (the historic behaviour). Set SCADABRIDGE_IMAGE (same variable docker/build.sh
|
||||
# tags with) to run env2 on its own build, e.g. SCADABRIDGE_IMAGE=scadabridge:env2-116,
|
||||
# so an env2 rebuild never retags the image the primary rig's containers run.
|
||||
services:
|
||||
central-a:
|
||||
image: scadabridge:latest
|
||||
image: ${SCADABRIDGE_IMAGE:-scadabridge:latest}
|
||||
# An init process (tini) as PID 1, so a crashed dotnet process actually dies
|
||||
# instead of wedging the container at 100% CPU with the restart policy never
|
||||
# firing — mirrors docker/docker-compose.yml's init: true (ScadaBridge#34).
|
||||
@@ -75,7 +79,7 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
central-b:
|
||||
image: scadabridge:latest
|
||||
image: ${SCADABRIDGE_IMAGE:-scadabridge:latest}
|
||||
init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a
|
||||
# CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting +
|
||||
# actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed
|
||||
@@ -137,7 +141,7 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
site-x-a:
|
||||
image: scadabridge:latest
|
||||
image: ${SCADABRIDGE_IMAGE:-scadabridge:latest}
|
||||
init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a
|
||||
# CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting +
|
||||
# actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed
|
||||
@@ -166,7 +170,7 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
site-x-b:
|
||||
image: scadabridge:latest
|
||||
image: ${SCADABRIDGE_IMAGE:-scadabridge:latest}
|
||||
init: true # PID-1 crash wedge, ScadaBridge#34 — see central-a
|
||||
# CoordinatedShutdown needs cluster-leave (15s budget) + cluster-exiting +
|
||||
# actor-system-terminate + Serilog flush; the 10s SIGTERM default SIGKILLed
|
||||
|
||||
+9
-3
@@ -34,15 +34,21 @@ if [ -z "$SOURCE_REVISION" ] && git -C "$REPO_ROOT" rev-parse --git-dir >/dev/nu
|
||||
fi
|
||||
echo "Source revision: ${SOURCE_REVISION:-<none>}"
|
||||
|
||||
# Image tag. Default scadabridge:latest (what docker/docker-compose.yml runs). Set
|
||||
# SCADABRIDGE_IMAGE to build under another tag without touching :latest, e.g. so
|
||||
# docker-env2 can run a different build while another agent uses the primary rig
|
||||
# (ScadaBridge#116 env2 rollout). docker-env2/docker-compose.yml reads the same variable.
|
||||
IMAGE="${SCADABRIDGE_IMAGE:-scadabridge:latest}"
|
||||
|
||||
# Build from repo root (so COPY paths in Dockerfile resolve correctly)
|
||||
echo "Building scadabridge:latest image..."
|
||||
echo "Building ${IMAGE} image..."
|
||||
# Note: ${NUGET_ARGS[@]+...} guards against the "unbound variable" error that
|
||||
# set -u raises on bash 3.2 (macOS default) when expanding an empty array.
|
||||
docker build \
|
||||
-t scadabridge:latest \
|
||||
-t "${IMAGE}" \
|
||||
-f "$SCRIPT_DIR/Dockerfile" \
|
||||
${NUGET_ARGS[@]+"${NUGET_ARGS[@]}"} \
|
||||
--build-arg "SOURCE_REVISION=${SOURCE_REVISION}" \
|
||||
"$REPO_ROOT"
|
||||
|
||||
echo "Build complete: scadabridge:latest"
|
||||
echo "Build complete: ${IMAGE}"
|
||||
|
||||
+21
-11
@@ -20,7 +20,12 @@
|
||||
# singletons, and /health/active goes 200 on the survivor WHILE THE VICTIM
|
||||
# IS STILL DOWN. Budget ~25s + singleton hand-over + health-probe margin.
|
||||
#
|
||||
# DRILL_PAIR=central (default) | site-a | site-b | site-c — which 2-node pair.
|
||||
# DRILL_ENV=primary (default) | env2 — which docker stack. env2 is the second
|
||||
# stack in docker-env2/ (containers scadabridge-env2-*, Traefik :9100, single
|
||||
# site pair site-x); TRAEFIK_URL defaults to its LB accordingly.
|
||||
#
|
||||
# DRILL_PAIR=central (default) | site-a | site-b | site-c (primary) | site-x (env2)
|
||||
# — which 2-node pair.
|
||||
# Central health is on container port 5000 (also published as 9001/9002 and
|
||||
# routed by Traefik on 9000); a site's /health/* is on container port 8084 and
|
||||
# is NOT published, so the script probes every pair from a throwaway sidecar
|
||||
@@ -37,7 +42,13 @@
|
||||
# fresh incarnation (standby), then that exactly one node of the pair is active.
|
||||
set -euo pipefail
|
||||
|
||||
TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
|
||||
DRILL_ENV="${DRILL_ENV:-primary}"
|
||||
case "$DRILL_ENV" in
|
||||
primary) NAME_PREFIX="scadabridge-"; DEFAULT_TRAEFIK="http://localhost:9000"; SITE_PAIRS="site-a|site-b|site-c" ;;
|
||||
env2) NAME_PREFIX="scadabridge-env2-"; DEFAULT_TRAEFIK="http://localhost:9100"; SITE_PAIRS="site-x" ;;
|
||||
*) echo "ERROR: DRILL_ENV must be 'primary' or 'env2' (was '$DRILL_ENV')" >&2; exit 2 ;;
|
||||
esac
|
||||
TRAEFIK_URL="${TRAEFIK_URL:-$DEFAULT_TRAEFIK}"
|
||||
TIMEOUT_S="${TIMEOUT_S:-90}"
|
||||
REJOIN_TIMEOUT_S="${REJOIN_TIMEOUT_S:-180}"
|
||||
DRILL_MODE="${DRILL_MODE:-standby}"
|
||||
@@ -49,18 +60,17 @@ case "$DRILL_MODE" in
|
||||
standby|active) ;;
|
||||
*) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;;
|
||||
esac
|
||||
case "$DRILL_PAIR" in
|
||||
central) HEALTH_PORT=5000 ;;
|
||||
site-a|site-b|site-c) HEALTH_PORT=8084 ;;
|
||||
*) echo "ERROR: DRILL_PAIR must be central|site-a|site-b|site-c (was '$DRILL_PAIR')" >&2; exit 2 ;;
|
||||
esac
|
||||
if [ "$DRILL_PAIR" = central ]; then HEALTH_PORT=5000
|
||||
elif [[ "$DRILL_PAIR" =~ ^(${SITE_PAIRS})$ ]]; then HEALTH_PORT=8084
|
||||
else echo "ERROR: DRILL_PAIR must be central|${SITE_PAIRS} for DRILL_ENV=${DRILL_ENV} (was '$DRILL_PAIR')" >&2; exit 2
|
||||
fi
|
||||
case "$DRILL_ACTION" in
|
||||
kill|pause) ;;
|
||||
*) echo "ERROR: DRILL_ACTION must be 'kill' or 'pause' (was '$DRILL_ACTION')" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
NODE_A="scadabridge-${DRILL_PAIR}-a"
|
||||
NODE_B="scadabridge-${DRILL_PAIR}-b"
|
||||
NODE_A="${NAME_PREFIX}${DRILL_PAIR}-a"
|
||||
NODE_B="${NAME_PREFIX}${DRILL_PAIR}-b"
|
||||
|
||||
PROBE="scadabridge-drill-probe-$$"
|
||||
docker run -d --rm --name "$PROBE" --network scadabridge-net "$PROBE_IMAGE" sleep 3600 > /dev/null
|
||||
@@ -96,7 +106,7 @@ else
|
||||
VICTIM_SEL="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE")
|
||||
fi
|
||||
|
||||
echo "pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} active=${ACTIVE} victim=${VICTIM_SEL} survivor=${SURVIVOR}"
|
||||
echo "env=${DRILL_ENV} pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} active=${ACTIVE} victim=${VICTIM_SEL} survivor=${SURVIVOR}"
|
||||
HIT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
VICTIM="$VICTIM_SEL"
|
||||
docker "$DRILL_ACTION" "${VICTIM}" > /dev/null
|
||||
@@ -202,5 +212,5 @@ echo "Exactly one active node in 10/10 samples."
|
||||
|
||||
echo "Survivor downing/singleton evidence (last 20 matching log lines from ${SURVIVOR}):"
|
||||
docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Ei "leader-auto-down|auto-downing|-> Oldest|downing|marking.*down|removed" | tail -20 || true
|
||||
echo "SUMMARY pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} down_logged_s=${DOWN_S:-n/a} rejoin_s=${ELAPSED}"
|
||||
echo "SUMMARY env=${DRILL_ENV} pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} down_logged_s=${DOWN_S:-n/a} rejoin_s=${ELAPSED}"
|
||||
echo "Drill complete. Verify on the Health dashboard / Overview that both nodes show Up and exactly one is Primary."
|
||||
|
||||
Reference in New Issue
Block a user