From badc97afe85d1c0773b174f417ce6ee1f791a45a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:32:54 -0400 Subject: [PATCH 1/9] fix(docker): failover drill kills the STANDBY by default; active mode measures the registered keep-oldest outage instead of pretending recovery (plan R2-01 T1) --- docker/failover-drill.sh | 107 ++++++++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 19 deletions(-) diff --git a/docker/failover-drill.sh b/docker/failover-drill.sh index d600eb72..ca4c238a 100755 --- a/docker/failover-drill.sh +++ b/docker/failover-drill.sh @@ -1,44 +1,113 @@ #!/usr/bin/env bash # Failover drill against the running docker cluster (bash docker/deploy.sh first). -# Kills the ACTIVE central node (docker kill = SIGKILL, the hard-crash path that -# review 01 found had NO automatic recovery before the SBR downing provider was -# enabled) and measures how long Traefik takes to route to the survivor. +# +# ROUND-2 REWRITE (arch-review 01 round 2, N1). The original drill killed the +# ACTIVE central node — but under the unified oldest-member semantics the +# active node IS the oldest, i.e. the one crash two-node keep-oldest CANNOT +# survive (registered deferred user decision, master tracker 2026-07-08; +# SbrFailoverTests.cs XML doc). Two modes: +# +# DRILL_MODE=standby (default) — kills the STANDBY (younger) central node. +# The survivable direction: SBR downs the crashed member, the active node +# keeps its singletons, and Traefik routing never goes dark. PASS = the +# survivor logs the member removal within TIMEOUT_S (budget ~25s+: 10s +# failure detection + 15s stable-after) while /health/active stays up. +# +# DRILL_MODE=active — kills the ACTIVE (oldest) central node. THE EXPECTED +# OUTCOME IS A TOTAL CENTRAL OUTAGE: keep-oldest downs the partition +# without the oldest, so the younger survivor downs ITSELF (down-if-alone +# cannot help — the alone-oldest is dead and cannot down itself), and the +# self-downed survivor cannot re-form a cluster alone unless it is the +# FIRST seed (both nodes list central-a first; only the first seed may +# self-join). This mode measures the dark window and PASSes only when +# central recovers AFTER the victim container is restarted. It exists to +# make the registered gap observable — not to pretend it is covered. set -euo pipefail TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}" TIMEOUT_S="${TIMEOUT_S:-90}" +DRILL_MODE="${DRILL_MODE:-standby}" +OUTAGE_CONFIRM_S="${OUTAGE_CONFIRM_S:-60}" active_container() { if curl -sf -o /dev/null "http://localhost:9001/health/active"; then echo scadabridge-central-a elif curl -sf -o /dev/null "http://localhost:9002/health/active"; then echo scadabridge-central-b else echo "ERROR: no active central node found" >&2; exit 1; fi } +peer_of() { [ "$1" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a; } -VICTIM=$(active_container) -echo "Active central node: ${VICTIM} — killing it (SIGKILL, crash path)" +case "$DRILL_MODE" in + standby|active) ;; + *) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;; +esac + +ACTIVE=$(active_container) +if [ "$DRILL_MODE" = standby ]; then + VICTIM=$(peer_of "$ACTIVE"); SURVIVOR="$ACTIVE" +else + VICTIM="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE") +fi + +echo "mode=${DRILL_MODE} active=${ACTIVE} victim=${VICTIM} survivor=${SURVIVOR}" +KILL_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ) docker kill "${VICTIM}" > /dev/null START=$(date +%s) -echo "Waiting for the survivor to become active through Traefik (${TRAEFIK_URL}/health/active)..." +if [ "$DRILL_MODE" = standby ]; then + echo "Standby crash: waiting for ${SURVIVOR} to DOWN+REMOVE the dead member (SBR budget ~25s)..." + BLIPS=0 + while true; do + ELAPSED=$(( $(date +%s) - START )) + curl -sf -o /dev/null "${TRAEFIK_URL}/health/active" || BLIPS=$((BLIPS + 1)) + if docker logs --since "${KILL_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "marking.*node.*down|member removed|is removed"; then + echo "PASS: survivor removed the crashed member in ${ELAPSED}s (budget ~25s: 10s detection + 15s stable-after)." + echo "Active-node routing blips during the drill: ${BLIPS} (expected 0 — the active node was never touched)." + break + fi + if (( ELAPSED > TIMEOUT_S )); then + echo "FAIL: no downing/removal evidence on ${SURVIVOR} after ${ELAPSED}s — SBR did not act" >&2 + docker start "${VICTIM}" > /dev/null + exit 1 + fi + sleep 1 + done +else + echo "Active crash: EXPECTING a central outage (registered keep-oldest gap). Watching /health/active..." + DARK_STREAK=0 + while true; do + ELAPSED=$(( $(date +%s) - START )) + if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then DARK_STREAK=0; else DARK_STREAK=$((DARK_STREAK + 1)); fi + if (( DARK_STREAK >= 10 )); then + echo "Outage confirmed at ${ELAPSED}s: no active central node — the younger survivor self-downed" + echo "(keep-oldest downs the partition WITHOUT the oldest; this is the registered deferred gap)." + break + fi + if (( ELAPSED > OUTAGE_CONFIRM_S )); then + echo "NOTE: /health/active stayed reachable ${ELAPSED}s after killing the oldest — better than the" + echo "registered gap predicts. Do NOT celebrate: capture both nodes' logs and investigate before trusting it." + break + fi + sleep 1 + done +fi + +echo "Restarting ${VICTIM}..." +docker start "${VICTIM}" > /dev/null +RESTART_AT=$(date +%s) +echo "Waiting for central to be routable again through Traefik (${TRAEFIK_URL}/health/active)..." while true; do - ELAPSED=$(( $(date +%s) - START )) + ELAPSED=$(( $(date +%s) - RESTART_AT )) if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then - echo "PASS: failover complete in ${ELAPSED}s (design budget ~25s + Traefik health interval)" + echo "Recovered: an active central node is routable ${ELAPSED}s after the victim restart." break fi - if (( ELAPSED > TIMEOUT_S )); then - echo "FAIL: no active central node after ${ELAPSED}s — SBR/singleton handover did not recover" >&2 - docker start "${VICTIM}" > /dev/null + if (( ELAPSED > 120 )); then + echo "FAIL: central not routable 120s after restarting ${VICTIM}" >&2 exit 1 fi sleep 1 done -SURVIVOR=$([ "${VICTIM}" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a) -echo "Survivor singleton evidence (last 20 matching log lines from ${SURVIVOR}):" -docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|Downing|Removed" | tail -20 || true - -echo "Restarting ${VICTIM} and waiting for it to rejoin..." -docker start "${VICTIM}" > /dev/null -sleep 5 -echo "Drill complete. Verify on the Health dashboard that both nodes show Up and the survivor is Primary." +echo "Survivor singleton/downing evidence (last 20 matching log lines from ${SURVIVOR}):" +docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|downing|removed" | tail -20 || true +echo "Drill complete (${DRILL_MODE}). Verify on the Health dashboard that both nodes show Up and exactly one is Primary." From d536450611f2e2f1e2ff84d7f496f33fb6310db8 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:33:56 -0400 Subject: [PATCH 2/9] =?UTF-8?q?docs(cluster):=20per-direction=20failover?= =?UTF-8?q?=20truth=20=E2=80=94=20first-seed=20bootstrap=20constraint=20+?= =?UTF-8?q?=20operator=20recovery=20actions;=20drop=20the=20undeliverable?= =?UTF-8?q?=20~25s=20active-crash=20promise=20(plan=20R2-01=20T2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/README.md | 22 ++++++++++++------- .../Component-ClusterInfrastructure.md | 6 +++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/docker/README.md b/docker/README.md index 63530d93..f64f19bf 100644 --- a/docker/README.md +++ b/docker/README.md @@ -273,19 +273,25 @@ All test passwords are `password`. See `infra/glauth/config.toml` for the full l ### Automated failover drill (`failover-drill.sh`) ```bash -bash docker/failover-drill.sh +DRILL_MODE=standby bash docker/failover-drill.sh # default — survivable younger-node crash +DRILL_MODE=active bash docker/failover-drill.sh # oldest-node crash — measures the registered outage gap ``` -The scripted drill is the repeatable version of the manual steps below. It: +The scripted drill (`docker kill` = SIGKILL, the hard-crash path — a `docker stop` would take the graceful `CoordinatedShutdown` path and would not prove crash recovery) has **two modes**, because under the unified oldest-member semantics the *active* node IS the oldest, i.e. the one crash two-node keep-oldest cannot survive: -1. Finds the **active** central node (via each node's `/health/active`). -2. **`docker kill`s it — SIGKILL, the hard-crash path.** This is the exact scenario arch-review 01 found had *no* automatic recovery before the SBR downing provider was enabled (`Akka.Cluster.SBR.SplitBrainResolverProvider` is now named explicitly in HOCON); a `docker stop` would take the graceful `CoordinatedShutdown` path instead and would not prove crash recovery. -3. Polls Traefik (`http://localhost:9000/health/active`, override with `TRAEFIK_URL`) until the survivor is routable, printing the observed failover time. Fails after `TIMEOUT_S` (default 90s). -4. Dumps the survivor's singleton/oldest/downing log evidence, then restarts the victim so it **rejoins as a fresh incarnation** (Task 20's recovery loop). +- **`DRILL_MODE=standby` (default) — kills the STANDBY (younger) central node.** The survivable direction: SBR downs the crashed member and the active node keeps its singletons. Expected result: **no routing outage at all** (the active node is never touched, so `/health/active` blips = 0) and member removal on the survivor within **~25s** (10s failure-detection threshold + 15s stable-after; the 2s heartbeat interval is not additive). PASS = the survivor logs the member removal within `TIMEOUT_S` (default 90s) while routing stays up. +- **`DRILL_MODE=active` — kills the ACTIVE (oldest) central node.** Expected result: a **total central outage** until the victim container is restarted — this is the registered deferred keep-oldest decision (master tracker 2026-07-08): keep-oldest downs the partition *without* the oldest, so the younger survivor downs itself, and it cannot re-form a cluster alone (see the seed-node constraint below). The drill confirms the dark window, then recovery within ~2 min of restarting the victim. The mode exists to make the registered gap *observable*, not to pretend it is covered. -Expected failover: **~25s** (2s heartbeat + 10s detection + 15s stable-after) **plus the Traefik ~5s health-check interval**. The drill exercises S1 (SBR downing on hard crash), S3 (single active node routed through Traefik), and the Task 20 restart/rejoin contract. Requires a running cluster (`bash docker/deploy.sh`) and `curl` + `docker` on the host. +The drill exercises S1 (SBR downing on hard crash), S3 (single active node routed through Traefik), and the Task 20 restart/rejoin contract. Requires a running cluster (`bash docker/deploy.sh`) and `curl` + `docker` on the host. -> **Observed failover time:** _run after next deploy and record here_ (the script was validated with `bash -n`; no live cluster was up when it was committed). +**Seed-node bootstrap constraint.** Only the FIRST seed in `Cluster:SeedNodes` may self-join to form a *new* cluster. Both central nodes list `scadabridge-central-a` first (`docker/central-node-a/appsettings.Central.json`, `docker/central-node-b/appsettings.Central.json`), so a lone restarted `central-b` (with `central-a` still down) loops on `InitJoin` forever — it never reaches `Up`, and `/health/active` never returns 200. Operator recovery actions: **(1)** restart the dead first-seed node (`central-a`) — preferred; or **(2)** restart the survivor with a self-first seed override (env `ScadaBridge__Cluster__SeedNodes__0=akka.tcp://scadabridge@:8081`, `ScadaBridge__Cluster__SeedNodes__1=`). The repo deliberately does NOT ship self-first ordering per node: with *both* nodes self-first, a simultaneous cold start can let each self-join independently → two one-node clusters that never merge (the cold-start split-brain the identical-seed-order convention exists to prevent). The real remedy is the pending keep-oldest topology/strategy decision (deferred, owner: user). + +> **Observed results** (recorded by plan R2-01 T3, cluster @ commit ): +> +> | Direction (`DRILL_MODE`) | Outcome | Measured | +> |--------------------------|---------|----------| +> | `standby` (younger-node crash) | _pending T3_ | _pending T3_ | +> | `active` (oldest-node crash) | _pending T3_ | _pending T3_ | ### Central Failover diff --git a/docs/requirements/Component-ClusterInfrastructure.md b/docs/requirements/Component-ClusterInfrastructure.md index c16f1a86..8b63b08b 100644 --- a/docs/requirements/Component-ClusterInfrastructure.md +++ b/docs/requirements/Component-ClusterInfrastructure.md @@ -108,9 +108,11 @@ When a node downs itself (via `down-if-alone`, or any other SBR decision), the r 1. Self-down ⇒ `CoordinatedShutdown` ⇒ `ActorSystem` termination. 2. The Host watches `ActorSystem.WhenTerminated`; a termination that is **not** the host's own `StopAsync` triggers `IHostApplicationLifetime.StopApplication()`, so **the process exits**. 3. The service supervisor restarts it — docker `restart: unless-stopped`, or Windows service recovery actions (`sc.exe failure … restart/…`). -4. The restarted process rejoins as a **fresh incarnation** (the keep-oldest resolver handles the rejoin cleanly; there is no stale membership to reconcile). +4. The restarted process rejoins as a **fresh incarnation** (the keep-oldest resolver handles the rejoin cleanly; there is no stale membership to reconcile) — **but only while a peer still holding cluster state is reachable**. A lone restarted node that is *not* the first seed cannot re-form a cluster on its own (see the seed-node bootstrap constraint below); it waits for its peer. -The docker failover drill exercises this loop end-to-end (kill the active node, assert singleton migration + clean rejoin). +**Seed-node bootstrap constraint.** Only the FIRST seed listed in `Cluster:SeedNodes` may self-join to form a *new* cluster. All nodes list the same first seed (e.g. `scadabridge-central-a`), so a lone restarted non-first-seed node (with the first seed still down) loops on `InitJoin` forever — never `Up`, never routable. This is why the two-node keep-oldest **oldest/active-node crash is a total-outage gap**: after the oldest dies the younger survivor self-downs, and it cannot re-bootstrap alone. Recovery is operator-driven — either restart the dead first-seed node (preferred) or restart the survivor with a self-first seed override (`ScadaBridge__Cluster__SeedNodes__0` = self, `__1` = peer). The repo does not ship self-first ordering per node: with both nodes self-first a simultaneous cold start risks two independent one-node clusters that never merge. Removing the gap itself is the **registered deferred keep-oldest topology/strategy decision** (master tracker 2026-07-08, owner: user). + +The docker failover drill (`docker/failover-drill.sh`) exercises this per direction: `standby` mode proves SBR downing + singleton continuity on the oldest; `active` mode measures the registered total-outage gap and the recovery-on-restart path. ## Single-Node Operation From 30356803e77f9afb97bae8c9cfd7056e6d2dc68d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:34:24 -0400 Subject: [PATCH 3/9] =?UTF-8?q?docs(docker):=20mark=20failover-drill=20res?= =?UTF-8?q?ults=20table=20PENDING=20=E2=80=94=20live=20drill=20run=20(plan?= =?UTF-8?q?=20R2-01=20T3)=20deferred=20to=20avoid=20disrupting=20the=20run?= =?UTF-8?q?ning=20docker=20cluster?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/README.md b/docker/README.md index f64f19bf..1920f3d3 100644 --- a/docker/README.md +++ b/docker/README.md @@ -286,12 +286,14 @@ The drill exercises S1 (SBR downing on hard crash), S3 (single active node route **Seed-node bootstrap constraint.** Only the FIRST seed in `Cluster:SeedNodes` may self-join to form a *new* cluster. Both central nodes list `scadabridge-central-a` first (`docker/central-node-a/appsettings.Central.json`, `docker/central-node-b/appsettings.Central.json`), so a lone restarted `central-b` (with `central-a` still down) loops on `InitJoin` forever — it never reaches `Up`, and `/health/active` never returns 200. Operator recovery actions: **(1)** restart the dead first-seed node (`central-a`) — preferred; or **(2)** restart the survivor with a self-first seed override (env `ScadaBridge__Cluster__SeedNodes__0=akka.tcp://scadabridge@:8081`, `ScadaBridge__Cluster__SeedNodes__1=`). The repo deliberately does NOT ship self-first ordering per node: with *both* nodes self-first, a simultaneous cold start can let each self-join independently → two one-node clusters that never merge (the cold-start split-brain the identical-seed-order convention exists to prevent). The real remedy is the pending keep-oldest topology/strategy decision (deferred, owner: user). -> **Observed results** (recorded by plan R2-01 T3, cluster @ commit ): +> **Observed results** (plan R2-01 T3): +> +> **PENDING — live drill not yet executed** (requires a dedicated docker cluster rebuild; deferred to avoid disrupting the running environment). The two-mode drill script and its per-direction PASS criteria are in place; the empirical timings below are to be filled by running `DRILL_MODE=standby` then `DRILL_MODE=active` against a freshly-deployed cluster and recording the one-line summaries + the cluster commit SHA. > > | Direction (`DRILL_MODE`) | Outcome | Measured | > |--------------------------|---------|----------| -> | `standby` (younger-node crash) | _pending T3_ | _pending T3_ | -> | `active` (oldest-node crash) | _pending T3_ | _pending T3_ | +> | `standby` (younger-node crash) | _pending live run_ | _pending live run_ | +> | `active` (oldest-node crash) | _pending live run_ | _pending live run_ | ### Central Failover From 65e0f770d182ab3261b69dca8fc72190c33bb815 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:38:19 -0400 Subject: [PATCH 4/9] =?UTF-8?q?test(failover):=20unskip=20FailoverTimingTe?= =?UTF-8?q?sts=20on=20the=20two-node=20rig=20at=20production=20timings=20?= =?UTF-8?q?=E2=80=94=20measured=2033.7s=20vs=20~25s=20design=20envelope=20?= =?UTF-8?q?(plan=20R2-01=20T4;=20covers=20report-08=20NF2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Cluster/TwoNodeClusterFixture.cs | 18 ++-- .../Failover/FailoverTimingTests.cs | 83 +++++++++++++------ ...MOM.WW.ScadaBridge.PerformanceTests.csproj | 2 + 3 files changed, 73 insertions(+), 30 deletions(-) diff --git a/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs b/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs index 3d1c94da..88bded93 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs @@ -13,6 +13,10 @@ namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster; /// singleton host). CrashNode() terminates a node WITHOUT cluster-leave /// (coordinated-shutdown by-terminate disabled on both nodes) to simulate /// a hard crash — the scenario review 01 found untested. +/// +/// Timing knobs default to scaled test values; pass production values +/// (2s heartbeat / 10s failure-detection / 15s stable-after) to measure the +/// real failover envelope (FailoverTimingTests). /// public sealed class TwoNodeClusterFixture : IAsyncDisposable { @@ -23,21 +27,23 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable public static async Task StartAsync( string role = "Central", TimeSpan? stableAfter = null, - int? portA = null, int? portB = null) + int? portA = null, int? portB = null, + TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null) { var f = new TwoNodeClusterFixture(); f.PortA = portA ?? GetFreeTcpPort(); f.PortB = portB ?? GetFreeTcpPort(); - f.NodeA = f.StartNode(f.PortA, role, stableAfter); + f.NodeA = f.StartNode(f.PortA, role, stableAfter, heartbeatInterval, failureDetectionThreshold); await WaitForMembersUp(f.NodeA, 1, TimeSpan.FromSeconds(20)); - f.NodeB = f.StartNode(f.PortB, role, stableAfter); + f.NodeB = f.StartNode(f.PortB, role, stableAfter, heartbeatInterval, failureDetectionThreshold); await WaitForMembersUp(f.NodeA, 2, TimeSpan.FromSeconds(20)); await WaitForMembersUp(f.NodeB, 2, TimeSpan.FromSeconds(20)); return f; } /// Starts a node from production HOCON; used by StartAsync and by restart-scenarios. - public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null) + public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null, + TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null) { var nodeOptions = new NodeOptions { Role = role, NodeHostname = "127.0.0.1", RemotingPort = port }; var clusterOptions = new ClusterOptions @@ -49,8 +55,8 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable }, SplitBrainResolverStrategy = "keep-oldest", StableAfter = stableAfter ?? TimeSpan.FromSeconds(3), - HeartbeatInterval = TimeSpan.FromMilliseconds(500), - FailureDetectionThreshold = TimeSpan.FromSeconds(2), + HeartbeatInterval = heartbeatInterval ?? TimeSpan.FromMilliseconds(500), + FailureDetectionThreshold = failureDetectionThreshold ?? TimeSpan.FromSeconds(2), MinNrOfMembers = 1, DownIfAlone = true, }; diff --git a/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs b/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs index 9e0e12db..615532c6 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs @@ -1,33 +1,68 @@ +using System.Diagnostics; +using Akka.Actor; +using Akka.Cluster.Tools.Singleton; +using Xunit.Abstractions; +using ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster; + namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.Failover; /// -/// Failover-timing benchmark harness (placeholder). +/// Failover-timing measurement on the real two-node in-process rig +/// (TwoNodeClusterFixture, production BuildHocon) at PRODUCTION timings: +/// 2s heartbeat / 10s failure-detection threshold / 15s SBR stable-after — +/// the CLAUDE.md "total failover ~25s" design envelope. /// -/// Design envelope under test (CLAUDE.md "Cluster & Failover"): failure -/// detection 2s heartbeat / 10s threshold, SBR stable-after 15s, total -/// failover ~25s for a hard node loss. -/// -/// Measurement protocol (to be wired to the two-node cluster rig delivered -/// by PLAN-01 — see archreview/plans/PLAN-01-*.md; do not duplicate that rig -/// here): -/// 1. Form a real 2-node cluster (docker/ topology or Akka.Remote in-proc -/// multi-ActorSystem rig from PLAN-01). -/// 2. Confirm a cluster singleton (e.g. site DeploymentManager) is hosted -/// on node A; record T0. -/// 3. Hard-kill node A (process kill / ActorSystem.Abort — not -/// CoordinatedShutdown; graceful stop exercises a different path). -/// 4. Poll node B for singleton re-host; record T1 at first successful -/// response from the migrated singleton. -/// 5. Assert T1 - T0 <= 40s (25s design + margin) and report the number. +/// Measures the SURVIVABLE direction only: hard-crash of the YOUNGER node, +/// timed to the survivor's member REMOVAL (detection + stable-after + gossip) +/// with singleton continuity asserted on the oldest. The oldest/active-node +/// crash is NOT a recovery to time — two-node keep-oldest makes the younger +/// survivor down itself (total outage; registered deferred user decision, +/// see SbrFailoverTests XML doc + master tracker 2026-07-08). Covers overall +/// review P2-10 / report-08 NF2 and report-01 round-2 N1's measurement ask. /// -public class FailoverTimingTests +public class FailoverTimingTests(ITestOutputHelper output) { - [Trait("Category", "Performance")] - [Fact(Skip = "Requires the real two-node failover rig delivered by PLAN-01 " + - "(overall review P2-10). This class documents the measurement " + - "protocol; wire it to the rig when PLAN-01 lands.")] - public void HardKillFailover_SingletonRehostedWithinDesignEnvelope() + private sealed class EchoActor : ReceiveActor { - // Intentionally empty until the PLAN-01 rig exists. + public EchoActor() => ReceiveAny(msg => Sender.Tell(msg)); + } + + [Trait("Category", "Performance")] + [Fact] + public async Task HardKillOfYoungerNode_SbrRemovalWithinDesignEnvelope() + { + await using var cluster = await TwoNodeClusterFixture.StartAsync( + stableAfter: TimeSpan.FromSeconds(15), + heartbeatInterval: TimeSpan.FromSeconds(2), + failureDetectionThreshold: TimeSpan.FromSeconds(10)); + + // Singleton on the oldest (NodeA) — the continuity probe. + var manager = cluster.NodeA.ActorOf(ClusterSingletonManager.Props( + Props.Create(() => new EchoActor()), + PoisonPill.Instance, + ClusterSingletonManagerSettings.Create(cluster.NodeA).WithSingletonName("timing-probe")), + "timing-probe-singleton"); + var proxyA = cluster.NodeA.ActorOf(ClusterSingletonProxy.Props( + "/user/timing-probe-singleton", + ClusterSingletonProxySettings.Create(cluster.NodeA).WithSingletonName("timing-probe")), + "timing-probe-proxy"); + Assert.Equal("ping", await proxyA.Ask("ping", TimeSpan.FromSeconds(30))); + + var victim = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress; + var sw = Stopwatch.StartNew(); + await TwoNodeClusterFixture.CrashNode(cluster.NodeB); + await TwoNodeClusterFixture.WaitForMemberRemoved(cluster.NodeA, victim, TimeSpan.FromSeconds(60)); + sw.Stop(); + + output.WriteLine( + $"MEASURED: crash -> member removed on survivor: {sw.Elapsed.TotalSeconds:F1}s " + + "(design ~25s = 10s detection + 15s stable-after; +gossip margin)"); + + // Design envelope: >= stable-after (SBR must not act early), <= 25s design + 15s margin. + Assert.InRange(sw.Elapsed, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(40)); + + // Singleton continuity on the surviving oldest node. + Assert.Equal("ping-after-crash", + await proxyA.Ask("ping-after-crash", TimeSpan.FromSeconds(10))); } } diff --git a/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj b/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj index bd285f71..7ebabf02 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj +++ b/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj @@ -26,6 +26,8 @@ + + From e4765b2eede55550126bd67f106283c13b58444b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:42:38 -0400 Subject: [PATCH 5/9] =?UTF-8?q?docs(tracker):=20correct=20the=20wonder-app?= =?UTF-8?q?-vd03=20deferral=20record=20=E2=80=94=20artifact=20is=20present?= =?UTF-8?q?=20(gitignored);=20overlay=20edits=20applied=20on-disk=20by=20R?= =?UTF-8?q?2-01=20T5/T6=20(plan=20R2-01=20T7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- archreview/plans/00-MASTER-TRACKER.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archreview/plans/00-MASTER-TRACKER.md b/archreview/plans/00-MASTER-TRACKER.md index 02669a0e..ce39e353 100644 --- a/archreview/plans/00-MASTER-TRACKER.md +++ b/archreview/plans/00-MASTER-TRACKER.md @@ -76,7 +76,7 @@ Status values: ⬜ Not started · 🟨 In progress · ✅ Complete · ⏸ Blocke **Wave 1 — PLAN-02 COMPLETE (2026-07-08):** all 24 tasks delivered and **merged to `main`** (`ce239f6f`). Store-and-Forward: active-node delivery gate (`Func` seam), bounded/short-circuiting retry sweep, upsert-based standby applies + anti-entropy peer resync, WAL+busy_timeout, epoch-ms due-check, ordered single-reader audit-observer channel; `Notify.Send` enqueue-only (`deferToSweep`, `maxRetries:0`, corrupt-payload parks not discards); gRPC channel keying + DropOldest debug-stream eviction counting + 5s application heartbeat; oldest-parked-age health field. Docker cluster rebuilt on merged image (Healthy, 5 singletons, corrected active-node semantics) and 3 sites seeded. Initiative total: **52 of 192** (sum of the Done column above). -**Wave 1 — PLAN-01 COMPLETE (2026-07-08):** all 23 tasks delivered on worktree branch `worktree-archreview-wave1-plan01` (**merged to `main` `ec47cb56` 2026-07-09; branch since removed**). Beyond the P0/early-Wave-1 spine (SBR downing + two-node rig T1/T3/T4; oldest-member active-node unification via `ClusterActivityEvaluator` T5–T8; `CentralSingletonRegistrar` drains T2/T9/T10; health-report acking T11–T12), Wave 1 also landed: metrics-staleness + status-transition state machine and its dashboard surface (T13/T14), deleted-site eviction from the aggregator (T18), seed-vs-metrics-port validator + `AllowSingleNodeCluster` + docker `stop_grace_period` (T15–T17), rate-limited dead-letter warnings (T19), the down-if-alone process-exit recovery loop (T20), the `failover-drill.sh` SIGKILL drill (T21), and the REQ-HOST-6 + Traefik/Host doc reconciliation with unused-Akka.Hosting-package removal (T22/T23). PLAN-01 now **23 / 23**. Initiative total: **30 of 192**. Each task TDD + pathspec-committed; full solution builds clean (0 warnings). Three deploy-artifact sub-edits (`deploy/wonder-app-vd03/` appsettings + install.ps1, Tasks 16/20/23) were deferred because that production artifact is not tracked in this repo — logged in the deferred register below. +**Wave 1 — PLAN-01 COMPLETE (2026-07-08):** all 23 tasks delivered on worktree branch `worktree-archreview-wave1-plan01` (**merged to `main` `ec47cb56` 2026-07-09; branch since removed**). Beyond the P0/early-Wave-1 spine (SBR downing + two-node rig T1/T3/T4; oldest-member active-node unification via `ClusterActivityEvaluator` T5–T8; `CentralSingletonRegistrar` drains T2/T9/T10; health-report acking T11–T12), Wave 1 also landed: metrics-staleness + status-transition state machine and its dashboard surface (T13/T14), deleted-site eviction from the aggregator (T18), seed-vs-metrics-port validator + `AllowSingleNodeCluster` + docker `stop_grace_period` (T15–T17), rate-limited dead-letter warnings (T19), the down-if-alone process-exit recovery loop (T20), the `failover-drill.sh` SIGKILL drill (T21), and the REQ-HOST-6 + Traefik/Host doc reconciliation with unused-Akka.Hosting-package removal (T22/T23). PLAN-01 now **23 / 23**. Initiative total: **30 of 192**. Each task TDD + pathspec-committed; full solution builds clean (0 warnings). Three deploy-artifact sub-edits (`deploy/wonder-app-vd03/` appsettings + install.ps1, Tasks 16/20/23) were deferred on a mistaken not-in-repo rationale; corrected and applied on-disk by PLAN-R2-01 (2026-07-12) — see the Deferred-during-Wave-1 entry. **Wave 1 — PLAN-03 COMPLETE (2026-07-09):** all 26 tasks delivered and **merged to `main`** (`c7a0c922`). Site Runtime & DCL hardening across nine TDD batches: reconnect/backoff leak fixes and the `DeployCompileValidator` compile gate, DCL adapter lifecycle + alarm-filter overwrite warning (S10) + observed adapter dispose on stop (S9), site-node cert reconcile (CertStoreActor trusted-store export → `DeploymentManagerActor` MemberUp-driven additive push), replicated config-fetch retry (`ConfigFetchRetryCount`), invariant-culture conditional-trigger fallback (C6), bounded/DropOldest event-channel comment fix (C2), and the SiteRuntime/DCL requirements-doc sync. Three low-severity items deferred with rationale (see PLAN-03 coverage table). Full CentralUI/SiteRuntime/DCL suites green. Initiative total: **78 of 192** (sum of the Done column above). Two out-of-band fixes also landed post-merge: PLAN-07's SecuredWrites `IBrowseService` bUnit-harness gap (resolved 2026-07-09, `9049690b`) — a pre-existing failure surfaced during PLAN-03's full-suite run, not a plan task. @@ -195,7 +195,7 @@ Consolidated in **PLAN-08 Task 11** → `docs/plans/2026-07-08-deferred-work-reg ### Deferred during Wave 1 PLAN-01 execution (2026-07-08) -- **`deploy/wonder-app-vd03/` overlay edits not applied — that production deploy artifact is not tracked in this repo.** PLAN-01 Tasks 16, 20, and 23 each specified a small edit to the on-host deployment overlay (`appsettings.Central.json`: add `AllowSingleNodeCluster: true` + drop the phantom seed, add `NodeName: central-a`; `install.ps1`: add `sc.exe failure` service-recovery actions for the down-if-alone restart contract). Only the *deployment-record* doc (`docs/deployment/deployment-records/2026-06-27-wonder-app-vd03.md`) is version-controlled; the `deploy/wonder-app-vd03/` artifact directory itself is not present in the repo (production config kept out of source control). The **code/validator/doc** halves of all three tasks shipped and are what make the features usable (`AllowSingleNodeCluster` flag + validator, the `IHostApplicationLifetime` process-exit watchdog, the oldest-member Traefik/Host doc reconciliation + TLS roadmap note). **Owner: whoever maintains the on-host artifact** — apply the three overlay edits out-of-band on `wonder-app-vd03` (see the referenced task steps); without `NodeName` every audit row from that deployment stamps a NULL `SourceNode`, and without the service-recovery actions a self-downed node will not auto-restart. +- **`deploy/wonder-app-vd03/` overlay edits — applied on-disk 2026-07-12 by PLAN-R2-01 Tasks 5-6 (the earlier not-in-repo deferral rationale was factually wrong).** PLAN-01 Tasks 16, 20, and 23 each specified a small edit to the on-host deployment overlay (`appsettings.Central.json`/`appsettings.Site.json`: add `AllowSingleNodeCluster: true` + drop the phantom seed, add `NodeName` central-a/node-a; `install.ps1`: complete the recovery actions with `sc.exe failureflag` for the down-if-alone restart contract). The original deferral claimed the artifact directory "is not present in the repo" — this was **false**: `deploy/wonder-app-vd03/` **is present in the working tree, merely gitignored** (`.gitignore:48: /deploy/` — production config kept out of version control, not absent), and was actively maintained here (PLAN-07 edited its `_comment_Security`). Correcting the record: the *presence* claim was wrong; only the *version-control-tracking* is intentionally absent. PLAN-R2-01 Tasks 5-6 applied the three overlay edits directly on-disk (no `git add` — the artifact stays gitignored; `sc.exe failure` service-recovery actions were already present on disk, so T6 only added the missing `failureflag`). The **code/validator/doc** halves of all three original tasks had already shipped (`AllowSingleNodeCluster` flag + validator, the `IHostApplicationLifetime` process-exit watchdog, the oldest-member Traefik/Host doc reconciliation + TLS roadmap note). **Remaining owner action:** sync the artifact to `wonder-app-vd03` and restart services — appsettings take effect on restart; `failureflag` on re-running `install.ps1` §7 (or running the two `sc.exe` lines manually) — per the RUNBOOK troubleshooting row added by T6. Without `NodeName` every audit row from that deployment stamps a NULL `SourceNode`; without the failure-actions flag a self-downed (non-crash-exit) node will not auto-restart. ### Plan deviations during Wave 1 PLAN-03 execution (2026-07-09) From 11efe4be059749395f12619e06acffbb4df26c8f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:45:22 -0400 Subject: [PATCH 6/9] =?UTF-8?q?fix(health):=20flag=20metrics-stale=20for?= =?UTF-8?q?=20never-reported=20sites=20via=20FirstSeenAt=20anchor=20?= =?UTF-8?q?=E2=80=94=20null=20LastReportReceivedAt=20no=20longer=20skips?= =?UTF-8?q?=20the=20check=20(plan=20R2-01=20T8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Component-HealthMonitoring.md | 2 +- .../CentralHealthAggregator.cs | 16 +++++++--- .../SiteHealthState.cs | 11 +++++++ .../CentralHealthAggregatorTests.cs | 31 +++++++++++++++++++ 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/docs/requirements/Component-HealthMonitoring.md b/docs/requirements/Component-HealthMonitoring.md index 6b612d4c..1d58ee2e 100644 --- a/docs/requirements/Component-HealthMonitoring.md +++ b/docs/requirements/Component-HealthMonitoring.md @@ -44,7 +44,7 @@ Site clusters (metric collection and reporting). Central cluster (aggregation an - Each report is a **flat snapshot** containing the current values of all monitored metrics, a **monotonic sequence number**, and the **report timestamp** from the site. Central replaces the previous state for that site only if the incoming sequence number is higher than the last received — this prevents stale reports (e.g., delayed in transit or from a pre-failover node) from overwriting newer state. - Central tracks two **distinct** signals per site — do not conflate them: - **Liveness (online/offline)**: driven by the **last signal of any kind** (a full report *or* a transport heartbeat, the latter arriving every ~5s from any reachable site node). If central receives no signal within `OfflineTimeout` (default: **60 seconds**), the site is marked **offline**. Because heartbeats are frequent, this only fires on genuine total loss of contact — not during a single-node failover, when the surviving node keeps heartbeating. The synthetic *central* self-report site has no heartbeat source and uses the longer `CentralOfflineTimeout` (default: **3 minutes**) so a single missed self-report does not flap it offline. - - **Metrics staleness**: an **online** site (heartbeats still arriving) that has produced no full **report** within `MetricsStaleTimeout` (default: **2 minutes**) is flagged **metrics stale**. This surfaces the failure mode where a site's report pipeline (`HealthReportSender`) has died but the node is otherwise reachable — previously such a site showed "online with frozen metrics forever". `MetricsStaleTimeout` must be positive and no shorter than the report interval. + - **Metrics staleness**: an **online** site (heartbeats still arriving) that has produced no full **report** within `MetricsStaleTimeout` (default: **2 minutes**) is flagged **metrics stale**. This surfaces the failure mode where a site's report pipeline (`HealthReportSender`) has died but the node is otherwise reachable — previously such a site showed "online with frozen metrics forever". A site that has **never** delivered a report is anchored on its first-contact time (`FirstSeenAt`), so a report pipeline dead from first boot is also flagged after `MetricsStaleTimeout` (previously such a site showed online-with-no-metrics indefinitely). `MetricsStaleTimeout` must be positive and no shorter than the report interval. - **Online recovery**: when central receives a health report from a site that was marked offline, the site is automatically marked **online** and the metrics-stale flag cleared (a fresh report proves the pipeline is alive); a heartbeat alone also restores **online** but leaves the metrics-stale flag untouched. No manual acknowledgment required. - **Status-transition timestamp**: each online↔offline flip stamps `LastStatusChangeAt`, so the UI can answer "since when has this site been offline/online" rather than only "is it offline now". Report/heartbeat traffic that leaves the status unchanged does not move it. diff --git a/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthAggregator.cs b/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthAggregator.cs index 1cd236e1..811a6e5f 100644 --- a/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthAggregator.cs +++ b/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthAggregator.cs @@ -47,6 +47,7 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat LatestReport = report, LastReportReceivedAt = now, LastHeartbeatAt = now, + FirstSeenAt = now, LastSequenceNumber = report.SequenceNumber, IsOnline = true, IsMetricsStale = false @@ -117,6 +118,7 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat LatestReport = null, LastReportReceivedAt = null, LastHeartbeatAt = receivedAt, + FirstSeenAt = receivedAt, LastSequenceNumber = 0, IsOnline = true }; @@ -297,16 +299,22 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat // when no report has arrived within MetricsStaleTimeout. Distinct from // offline — a site whose HealthReportSender crashed keeps heartbeating // and would otherwise show "online with frozen metrics forever". - if (state.LastReportReceivedAt is { } lastReport - && now - lastReport > _options.MetricsStaleTimeout + // A site that has NEVER reported (heartbeat-only registration, + // LastReportReceivedAt == null) anchors on FirstSeenAt instead — + // otherwise a pipeline dead from first boot is never flagged (N3). + var reportAnchor = state.LastReportReceivedAt ?? state.FirstSeenAt; + if (reportAnchor is { } anchor + && now - anchor > _options.MetricsStaleTimeout && !state.IsMetricsStale) { var stale = state with { IsMetricsStale = true }; if (_siteStates.TryUpdate(kvp.Key, stale, state)) { _logger.LogWarning( - "Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s (timeout: {Timeout}s)", - state.SiteId, (now - lastReport).TotalSeconds, _options.MetricsStaleTimeout.TotalSeconds); + "Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s{NeverNote} (timeout: {Timeout}s)", + state.SiteId, (now - anchor).TotalSeconds, + state.LastReportReceivedAt is null ? " (never reported since first contact)" : string.Empty, + _options.MetricsStaleTimeout.TotalSeconds); } // CAS loss ⇒ a fresh report/heartbeat swapped in ⇒ correct to skip. } diff --git a/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthState.cs b/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthState.cs index 3ef9c184..7767aef2 100644 --- a/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthState.cs +++ b/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthState.cs @@ -40,6 +40,17 @@ public sealed record SiteHealthState /// public DateTimeOffset LastHeartbeatAt { get; init; } + /// + /// Gets the instant this site was first observed by the aggregator (via a + /// report or a heartbeat); null only for states not built by the + /// aggregator (hand-constructed fixtures). Anchors metrics-staleness for a + /// site that has NEVER delivered a report (review 01 round-2 N3): with + /// null the staleness check previously + /// skipped the site forever. cannot anchor + /// this — every heartbeat advances it. + /// + public DateTimeOffset? FirstSeenAt { get; init; } + /// Gets the sequence number of the last accepted health report, used to reject out-of-order duplicates. public long LastSequenceNumber { get; init; } /// Gets a value indicating whether the site is currently considered online. diff --git a/tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/CentralHealthAggregatorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/CentralHealthAggregatorTests.cs index bc340ef6..25c3f8fb 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/CentralHealthAggregatorTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/CentralHealthAggregatorTests.cs @@ -513,4 +513,35 @@ public class CentralHealthAggregatorTests aggregator.MarkHeartbeat("site-a", time.GetUtcNow()); Assert.NotEqual(offlineAt, aggregator.GetSiteState("site-a")!.LastStatusChangeAt); } + + /// + /// Round-2 N3: a site known only via heartbeats (LastReportReceivedAt == null) + /// previously skipped the staleness check forever — a report pipeline dead + /// from FIRST BOOT showed "online with no metrics" indefinitely. FirstSeenAt + /// anchors the window instead (LastHeartbeatAt cannot: heartbeats advance it). + /// + [Fact] + public void HeartbeatOnlySite_NeverReported_IsFlaggedMetricsStale() + { + var (aggregator, time) = NewAggregator(metricsStaleTimeout: TimeSpan.FromMinutes(2)); + aggregator.MarkHeartbeat("site-a", time.GetUtcNow()); // registered via heartbeat only + time.Advance(TimeSpan.FromMinutes(3)); + aggregator.MarkHeartbeat("site-a", time.GetUtcNow()); // still heartbeating => online + aggregator.CheckForOfflineSites(); + var state = aggregator.GetSiteState("site-a")!; + Assert.True(state.IsOnline); // liveness unchanged + Assert.True(state.IsMetricsStale); // was impossible before the fix + } + + /// A heartbeat-only site inside the window is NOT falsely flagged. + [Fact] + public void HeartbeatOnlySite_WithinStaleWindow_IsNotFlagged() + { + var (aggregator, time) = NewAggregator(metricsStaleTimeout: TimeSpan.FromMinutes(2)); + aggregator.MarkHeartbeat("site-a", time.GetUtcNow()); + time.Advance(TimeSpan.FromMinutes(1)); + aggregator.MarkHeartbeat("site-a", time.GetUtcNow()); + aggregator.CheckForOfflineSites(); + Assert.False(aggregator.GetSiteState("site-a")!.IsMetricsStale); + } } From c2707fdcf863f62ba121e1460f724a6c25feb1bc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:46:18 -0400 Subject: [PATCH 7/9] =?UTF-8?q?docs(health):=20CentralHealthReportLoop=20c?= =?UTF-8?q?omments=20say=20oldest-member,=20not=20cluster=20leader=20?= =?UTF-8?q?=E2=80=94=20stop=20re-teaching=20the=20S2=20conflation=20(plan?= =?UTF-8?q?=20R2-01=20T9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CentralHealthReportLoop.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthReportLoop.cs b/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthReportLoop.cs index ab822374..bafe8366 100644 --- a/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthReportLoop.cs +++ b/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthReportLoop.cs @@ -9,9 +9,11 @@ namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring; /// Periodically builds a SiteHealthReport for the central cluster itself /// (siteId = ) and feeds it into the local /// CentralHealthAggregator so the UI can render central as another card -/// on /monitoring/health. Only the cluster leader (Primary) generates -/// reports — the standby's aggregator catches up on failover when it -/// becomes Primary and starts its own loop. +/// on /monitoring/health. Only the active node (Primary = the **oldest Up +/// member**, i.e. the singleton host — IClusterNodeProvider.SelfIsPrimary, +/// never Akka cluster leadership) generates reports; the standby's aggregator +/// catches up on failover when it becomes the oldest and its loop starts +/// reporting. /// public class CentralHealthReportLoop : BackgroundService { @@ -41,8 +43,9 @@ public class CentralHealthReportLoop : BackgroundService private readonly HealthMonitoringOptions _options; private readonly ILogger _logger; - // Seeded with Unix-ms so reports from a newly-elected central leader - // always sort after reports from any prior leader for siteId="central". + // Seeded with Unix-ms so reports from a node that has newly become active + // (oldest Up member) always sort after reports from any previously-active + // node for siteId="central". // The clock is read through the injected TimeProvider so the seeding is // deterministically testable. private long _sequenceNumber; From 80e5b3685258311968368a2c9618f3c09f6642fe Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:49:49 -0400 Subject: [PATCH 8/9] =?UTF-8?q?refactor(host):=20CentralSingletonRegistrar?= =?UTF-8?q?=20->=20SingletonRegistrar=20with=20optional=20role=20scope=20?= =?UTF-8?q?=E2=80=94=20unblocks=20site-singleton=20drains=20(plan=20R2-01?= =?UTF-8?q?=20T10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Actors/AkkaHostedService.cs | 20 +++++------ ...etonRegistrar.cs => SingletonRegistrar.cs} | 33 ++++++++++++------- .../CoordinatedShutdownTests.cs | 4 +-- ...rarTests.cs => SingletonRegistrarTests.cs} | 23 +++++++++++-- 4 files changed, 55 insertions(+), 25 deletions(-) rename src/ZB.MOM.WW.ScadaBridge.Host/Actors/{CentralSingletonRegistrar.cs => SingletonRegistrar.cs} (65%) rename tests/ZB.MOM.WW.ScadaBridge.Host.Tests/{CentralSingletonRegistrarTests.cs => SingletonRegistrarTests.cs} (67%) diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs index 7a7d8517..623414df 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs @@ -456,14 +456,14 @@ akka {{ var outboxAuditWriter = _serviceProvider .GetRequiredService(); - // All central singletons below are created through CentralSingletonRegistrar.Start, + // All central singletons below are created through SingletonRegistrar.Start, // which pins the actor to the active central node (ClusterSingletonManager), exposes a // stable address (ClusterSingletonProxy), and — crucially — registers a PhaseClusterLeave // GracefulStop drain task so an in-flight EF write drains before handover (review 01 // [Medium]: notification-outbox and audit-log-ingest previously had NO drain task). Names - // are unchanged ({name}-singleton / {name}-proxy). The two SITE singletons stay - // hand-rolled below because they are role-scoped (.WithRole(siteRole)). - var outbox = CentralSingletonRegistrar.Start( + // are unchanged ({name}-singleton / {name}-proxy). Site singletons use the same + // registrar with `role: siteRole` (Task 11 / round-2 N5). + var outbox = SingletonRegistrar.Start( _actorSystem!, "notification-outbox", Props.Create(() => new ZB.MOM.WW.ScadaBridge.NotificationOutbox.NotificationOutboxActor( _serviceProvider, @@ -496,7 +496,7 @@ akka {{ var auditIngestLogger = _serviceProvider.GetRequiredService() .CreateLogger(); - var auditIngest = CentralSingletonRegistrar.Start( + var auditIngest = SingletonRegistrar.Start( _actorSystem!, "audit-log-ingest", Props.Create(() => new ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogIngestActor( _serviceProvider, @@ -561,7 +561,7 @@ akka {{ var siteCallAuditOptions = _serviceProvider .GetRequiredService>().Value; - var siteCallAudit = CentralSingletonRegistrar.Start( + var siteCallAudit = SingletonRegistrar.Start( _actorSystem!, "site-call-audit", Props.Create(() => new ZB.MOM.WW.ScadaBridge.SiteCallAudit.SiteCallAuditActor( _serviceProvider, @@ -606,7 +606,7 @@ akka {{ var auditLogOptions = _serviceProvider .GetRequiredService>(); - CentralSingletonRegistrar.Start( + SingletonRegistrar.Start( _actorSystem!, "audit-log-purge", Props.Create(() => new ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeActor( _serviceProvider, @@ -629,7 +629,7 @@ akka {{ var auditReconClient = _serviceProvider .GetRequiredService(); - CentralSingletonRegistrar.Start( + SingletonRegistrar.Start( _actorSystem!, "site-audit-reconciliation", Props.Create(() => new ZB.MOM.WW.ScadaBridge.AuditLog.Central.SiteAuditReconciliationActor( auditReconSites, @@ -660,7 +660,7 @@ akka {{ var kpiHistoryLogger = _serviceProvider.GetRequiredService() .CreateLogger(); - CentralSingletonRegistrar.Start( + SingletonRegistrar.Start( _actorSystem!, "kpi-history-recorder", Props.Create(() => new ZB.MOM.WW.ScadaBridge.KpiHistory.KpiHistoryRecorderActor( _serviceProvider, @@ -686,7 +686,7 @@ akka {{ var pendingPurgeCommunicationOptions = _serviceProvider.GetRequiredService>(); - CentralSingletonRegistrar.Start( + SingletonRegistrar.Start( _actorSystem!, "pending-deployment-purge", Props.Create(() => new PendingDeploymentPurgeActor( _serviceProvider, diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Actors/CentralSingletonRegistrar.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Actors/SingletonRegistrar.cs similarity index 65% rename from src/ZB.MOM.WW.ScadaBridge.Host/Actors/CentralSingletonRegistrar.cs rename to src/ZB.MOM.WW.ScadaBridge.Host/Actors/SingletonRegistrar.cs index 9d38e9ff..e793938f 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/Actors/CentralSingletonRegistrar.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/Actors/SingletonRegistrar.cs @@ -5,21 +5,24 @@ using Microsoft.Extensions.Logging; namespace ZB.MOM.WW.ScadaBridge.Host.Actors; /// -/// Registers a central cluster singleton with the canonical naming scheme +/// Registers a cluster singleton with the canonical naming scheme /// ({name}-singleton / {name}-proxy), a PoisonPill termination /// message, and a PhaseClusterLeave drain task that GracefulStops the manager -/// so in-flight EF work completes before handover. Extracted from five -/// copy-pasted ~60-line blocks (review 01 [Low]) whose drift left the two -/// busiest singletons (notification-outbox, audit-log-ingest) without drain -/// tasks (review 01 [Medium]). +/// so in-flight EF (central) or SQLite (site) work completes before handover. +/// Extracted from copy-pasted ~60-line blocks (review 01 [Low]) whose drift +/// left the two busiest singletons (notification-outbox, audit-log-ingest) +/// without drain tasks (review 01 [Medium]). The optional role maps to +/// / +/// .WithRole, so role-scoped site singletons get the same drain +/// (review 01 round-2 N5). /// -internal static class CentralSingletonRegistrar +internal static class SingletonRegistrar { internal sealed record Handle(IActorRef Manager, IActorRef Proxy); /// - /// Creates the and proxy for a central - /// cluster singleton under the canonical naming scheme, and registers a + /// Creates the and proxy for a cluster + /// singleton under the canonical naming scheme, and registers a /// PhaseClusterLeave drain task that GracefulStops the manager on shutdown. /// /// The actor system to register the singleton on. @@ -27,19 +30,24 @@ internal static class CentralSingletonRegistrar /// The used to create the singleton's managed actor. /// Logger used to report drain progress/failures. /// Maximum time to wait for the drain task to complete; defaults to 10 seconds. + /// Optional cluster role to scope the singleton to (applied to both the manager and proxy settings). /// A carrying the singleton manager and proxy actor refs. internal static Handle Start( ActorSystem system, string name, Props singletonProps, ILogger logger, - TimeSpan? drainTimeout = null) + TimeSpan? drainTimeout = null, + string? role = null) { + var managerSettings = ClusterSingletonManagerSettings.Create(system).WithSingletonName(name); + if (role is not null) managerSettings = managerSettings.WithRole(role); + var manager = system.ActorOf( ClusterSingletonManager.Props( singletonProps, PoisonPill.Instance, - ClusterSingletonManagerSettings.Create(system).WithSingletonName(name)), + managerSettings), $"{name}-singleton"); var timeout = drainTimeout ?? TimeSpan.FromSeconds(10); @@ -61,10 +69,13 @@ internal static class CentralSingletonRegistrar return Akka.Done.Instance; }); + var proxySettings = ClusterSingletonProxySettings.Create(system).WithSingletonName(name); + if (role is not null) proxySettings = proxySettings.WithRole(role); + var proxy = system.ActorOf( ClusterSingletonProxy.Props( $"/user/{name}-singleton", - ClusterSingletonProxySettings.Create(system).WithSingletonName(name)), + proxySettings), $"{name}-proxy"); return new Handle(manager, proxy); diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs index b5943cfe..3e0fe0fb 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs @@ -51,9 +51,9 @@ public class CoordinatedShutdownTests // Review 01 [Medium]: notification-outbox and audit-log-ingest were the // only central singletons WITHOUT a cluster-leave drain task. All seven - // now go through CentralSingletonRegistrar.Start, which always adds the + // now go through SingletonRegistrar.Start, which always adds the // PhaseClusterLeave GracefulStop drain. - Assert.Contains("CentralSingletonRegistrar.Start(", content); + Assert.Contains("SingletonRegistrar.Start(", content); foreach (var name in new[] { "notification-outbox", "audit-log-ingest", "site-call-audit", "audit-log-purge", "site-audit-reconciliation", "kpi-history-recorder", "pending-deployment-purge" }) diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CentralSingletonRegistrarTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SingletonRegistrarTests.cs similarity index 67% rename from tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CentralSingletonRegistrarTests.cs rename to tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SingletonRegistrarTests.cs index 21ceb8be..938af66b 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CentralSingletonRegistrarTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SingletonRegistrarTests.cs @@ -5,7 +5,7 @@ using ZB.MOM.WW.ScadaBridge.Host.Actors; namespace ZB.MOM.WW.ScadaBridge.Host.Tests; -public class CentralSingletonRegistrarTests +public class SingletonRegistrarTests { private sealed class EchoActor : ReceiveActor { @@ -16,7 +16,7 @@ public class CentralSingletonRegistrarTests public async Task Start_CreatesManagerAndProxy_WithCanonicalNames_AndDrainTask() { using var system = CreateSingleNodeClusterSystem(); - var handle = CentralSingletonRegistrar.Start( + var handle = SingletonRegistrar.Start( system, "test-widget", Props.Create(() => new EchoActor()), NullLogger.Instance, drainTimeout: TimeSpan.FromSeconds(2)); @@ -36,6 +36,25 @@ public class CentralSingletonRegistrarTests Assert.True(handle.Manager.IsNobody() || system.WhenTerminated.IsCompleted); } + [Fact] + public async Task Start_WithRoleScope_CreatesRoleScopedSingleton_ThatAnswers() + { + // Round-2 N5: role scoping is what kept the two SITE singletons + // (deployment-manager, event-log-handler) hand-rolled and drain-less. + using var system = CreateSingleNodeClusterSystem(); // roles = ["Central"] + var handle = SingletonRegistrar.Start( + system, "role-widget", Props.Create(() => new EchoActor()), + NullLogger.Instance, drainTimeout: TimeSpan.FromSeconds(2), role: "Central"); + + Assert.EndsWith("/user/role-widget-singleton", handle.Manager.Path.ToString()); + Assert.EndsWith("/user/role-widget-proxy", handle.Proxy.Path.ToString()); + + // The node holds the role => manager instantiates the singleton and + // the role-scoped proxy resolves it. + var echo = await handle.Proxy.Ask("hi", TimeSpan.FromSeconds(20)); + Assert.Equal("hi", echo); + } + private static ActorSystem CreateSingleNodeClusterSystem() { var port = FreePort(); From 56d39b5fceda151f860a87a0a00ada69c37c0a35 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 11:01:44 -0400 Subject: [PATCH 9/9] =?UTF-8?q?fix(host):=20site=20singletons=20(deploymen?= =?UTF-8?q?t-manager,=20event-log-handler)=20via=20SingletonRegistrar=20?= =?UTF-8?q?=E2=80=94=20PhaseClusterLeave=20drains=20for=20site=20SQLite=20?= =?UTF-8?q?writes=20(plan=20R2-01=20T11)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Actors/AkkaHostedService.cs | 65 ++++++------------- .../CoordinatedShutdownTests.cs | 28 ++++++-- 2 files changed, 42 insertions(+), 51 deletions(-) diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs index 623414df..2e29f03a 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs @@ -797,36 +797,20 @@ akka {{ _logger.LogInformation("SiteReplicationActor created and S&F replication handler wired"); - // Create the Deployment Manager as a cluster singleton - var singletonProps = ClusterSingletonManager.Props( - singletonProps: Props.Create(() => new DeploymentManagerActor( - storage, - compilationService, - sharedScriptLibrary, - streamManager, - siteRuntimeOptionsValue, - dmLogger, - dclManager, - replicationActor, - siteHealthCollector, - _serviceProvider, - null, - deploymentConfigFetcher)), - terminationMessage: PoisonPill.Instance, - settings: ClusterSingletonManagerSettings.Create(_actorSystem!) - .WithRole(siteRole) - .WithSingletonName("deployment-manager")); - - _actorSystem!.ActorOf(singletonProps, "deployment-manager-singleton"); - - // Create a proxy for other actors to communicate with the singleton - var proxyProps = ClusterSingletonProxy.Props( - singletonManagerPath: "/user/deployment-manager-singleton", - settings: ClusterSingletonProxySettings.Create(_actorSystem) - .WithRole(siteRole) - .WithSingletonName("deployment-manager")); - - var dmProxy = _actorSystem.ActorOf(proxyProps, "deployment-manager-proxy"); + // Deployment Manager — role-scoped singleton via SingletonRegistrar + // (review 01 round-2 N5): previously hand-rolled with bare PoisonPill + // termination and NO PhaseClusterLeave drain, so in-flight SQLite + // writes (static overrides, native_alarm_state) could be cut off on + // graceful failover. Names unchanged: deployment-manager-singleton / + // deployment-manager-proxy. + var dm = SingletonRegistrar.Start( + _actorSystem!, "deployment-manager", + Props.Create(() => new DeploymentManagerActor( + storage, compilationService, sharedScriptLibrary, streamManager, + siteRuntimeOptionsValue, dmLogger, dclManager, replicationActor, + siteHealthCollector, _serviceProvider, null, deploymentConfigFetcher)), + _logger, role: siteRole); + var dmProxy = dm.Proxy; // Create SiteCommunicationActor for receiving messages from central var siteCommActor = _actorSystem.ActorOf( @@ -847,22 +831,11 @@ akka {{ var eventLogQueryService = _serviceProvider.GetService(); if (eventLogQueryService != null) { - var eventLogSingletonProps = ClusterSingletonManager.Props( - singletonProps: Props.Create(() => new SiteEventLogging.EventLogHandlerActor(eventLogQueryService)), - terminationMessage: PoisonPill.Instance, - settings: ClusterSingletonManagerSettings.Create(_actorSystem) - .WithRole(siteRole) - .WithSingletonName("event-log-handler")); - _actorSystem.ActorOf(eventLogSingletonProps, "event-log-handler-singleton"); - - var eventLogProxyProps = ClusterSingletonProxy.Props( - singletonManagerPath: "/user/event-log-handler-singleton", - settings: ClusterSingletonProxySettings.Create(_actorSystem) - .WithRole(siteRole) - .WithSingletonName("event-log-handler")); - var eventLogProxy = _actorSystem.ActorOf(eventLogProxyProps, "event-log-handler-proxy"); - - siteCommActor.Tell(new RegisterLocalHandler(LocalHandlerType.EventLog, eventLogProxy)); + var eventLog = SingletonRegistrar.Start( + _actorSystem, "event-log-handler", + Props.Create(() => new SiteEventLogging.EventLogHandlerActor(eventLogQueryService)), + _logger, role: siteRole); + siteCommActor.Tell(new RegisterLocalHandler(LocalHandlerType.EventLog, eventLog.Proxy)); } // Parked message handler — bridges Akka to StoreAndForwardService diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs index 3e0fe0fb..5b1a51ac 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/CoordinatedShutdownTests.cs @@ -61,11 +61,29 @@ public class CoordinatedShutdownTests Assert.Contains($"\"{name}\"", content); } - // No hand-rolled ClusterSingletonManager registrations remain in the - // CENTRAL branch. The only two left are the SITE singletons - // (deployment-manager, event-log-handler), which stay hand-rolled because - // they are role-scoped (.WithRole(siteRole)). - Assert.Equal(2, CountOccurrences(content, "ClusterSingletonManager.Props(")); + // No hand-rolled ClusterSingletonManager registrations remain anywhere: + // the two role-scoped SITE singletons now also go through the registrar + // (SingletonRegistrar.Start(..., role: siteRole)), so every singleton in + // the file is created through the registrar with a drain task. + Assert.Equal(0, CountOccurrences(content, "ClusterSingletonManager.Props(")); + } + + [Fact] + public void SiteSingletons_RegisterThroughRegistrarWithDrain() + { + var hostProjectDir = FindHostProjectDirectory(); + Assert.NotNull(hostProjectDir); + var content = File.ReadAllText(Path.Combine(hostProjectDir!, "Actors", "AkkaHostedService.cs")); + + // Round-2 N5: the two role-scoped SITE singletons previously stayed + // hand-rolled (bare PoisonPill, no PhaseClusterLeave drain). They now + // go through SingletonRegistrar.Start(..., role: siteRole), which + // always adds the GracefulStop drain — in-flight SQLite writes + // (static overrides, native_alarm_state) complete before handover. + Assert.Contains("\"deployment-manager\"", content); + Assert.Contains("\"event-log-handler\"", content); + Assert.Contains("role: siteRole", content); + Assert.Equal(0, CountOccurrences(content, "ClusterSingletonManager.Props(")); } private static int CountOccurrences(string haystack, string needle)