diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 39db2252..a8f63d13 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,3 +1,47 @@ +# ── Clustered secret replication: the pull-only gRPC hub (scadaproj#3) ───────── +# +# Central hosts the hub on its EXISTING h2c control-plane listener (CentralGrpcPort +# 8083, alongside CentralControlService); each site node sweeps it on an interval and +# writes what it pulls into its OWN local SQLite store. Nothing but ciphertext crosses +# the wire, so every participating node must resolve the SAME KEK. +# +# ENABLED ON FOUR NODES ONLY: the central pair (hub) and the site-a pair (followers). +# site-b and site-c are deliberately left without it, so the default-OFF posture is +# proven side by side on one rig — exactly as site-a is the rig's only LocalDb-replicated +# pair. A node with no Secrets__* override keeps the shipped appsettings default +# (Replication:Enabled=false, Mode=SqlServer) and composes a plain local store. +# +# ALL VALUES HERE ARE DEV-ONLY and committed under the same exception as the mesh PSKs +# and the ApiKeyPepper above: a local docker rig needs a working credential in source +# control to boot. Production supplies the KEK out of band (ZB_SECRETS_MASTER_KEY, never +# committed) and the hub token from appsettings/env — NEVER as a ${secret:} reference, +# since resolving one is what the hub exists to make possible. +x-secrets-hub-env: &secrets-hub-env + # DEV-ONLY KEK — NOT a real key. Identical on all four participating nodes: only + # ciphertext replicates, so a node with a different KEK fails closed on resolve with a + # kek_id mismatch that reads like corruption but is a deployment error. + ZB_SECRETS_MASTER_KEY: "zZiBWuoaVMbJmGXToLk9Lakw0iJozXoL/7Gxac3GwJ4=" + # The appsettings default is the relative "scadabridge-secrets.db", which resolves to + # /app — inside the image's writable layer, so it is destroyed by any container + # recreate and unreachable from the host. /app/data is the node's own mounted volume + # (the one LocalDb already uses on sites; added to the central pair for this). + Secrets__SqlitePath: "/app/data/scadabridge-secrets.db" + Secrets__Replication__Enabled: "true" + Secrets__Replication__Mode: "Grpc" + # DEV-ONLY shared bearer token — NOT a real secret. Presented by every follower and + # verified by the hub's fail-closed SecretsHubAuthInterceptor. Must be IDENTICAL on the + # hub and every follower; an unset token is a startup failure on both halves. + Secrets__GrpcHub__BearerToken: "secrets-hub-docker-dev-token" + +# Site half of the same section. NOTE the asymmetry with +# ScadaBridge:Communication:CentralGrpcEndpoints, which is a LIST that fails over across +# the central pair: the hub client dials a SINGLE endpoint, so a sweep against a stopped +# central-a stalls rather than failing over to central-b. That is survivable — the sweep +# is best-effort and the node keeps serving its full local last-known-good store — but +# secrets stop converging until central-a returns. +x-secrets-hub-site-env: &secrets-hub-site-env + Secrets__GrpcHub__Endpoint: "http://scadabridge-central-a:8083" + services: central-a: image: scadabridge:latest @@ -7,6 +51,8 @@ services: stop_grace_period: 30s container_name: scadabridge-central-a environment: + # Hub half of the pull-only gRPC secrets hub (anchor at the top of this file). + <<: *secrets-hub-env SCADABRIDGE_CONFIG: Central ASPNETCORE_ENVIRONMENT: Development ASPNETCORE_URLS: "http://+:5000" @@ -42,6 +88,11 @@ services: - "9013:8083" # gRPC control plane (CentralControlService, T1A.2) volumes: - ./central-node-a/appsettings.Central.json:/app/appsettings.Central.json:ro + # Added for the gRPC secrets hub: the node's local secret store lives at + # Secrets__SqlitePath=/app/data/scadabridge-secrets.db, so it needs the same + # per-node volume the site nodes already have. Without it the store sits in the + # image's writable layer and is destroyed by every container recreate. + - ./central-node-a/data:/app/data - ./central-node-a/logs:/app/logs networks: - scadabridge-net @@ -55,6 +106,8 @@ services: stop_grace_period: 30s container_name: scadabridge-central-b environment: + # Hub half of the pull-only gRPC secrets hub (anchor at the top of this file). + <<: *secrets-hub-env SCADABRIDGE_CONFIG: Central ASPNETCORE_ENVIRONMENT: Development ASPNETCORE_URLS: "http://+:5000" @@ -90,6 +143,11 @@ services: - "9014:8083" # gRPC control plane (CentralControlService, T1A.2) volumes: - ./central-node-b/appsettings.Central.json:/app/appsettings.Central.json:ro + # Added for the gRPC secrets hub: the node's local secret store lives at + # Secrets__SqlitePath=/app/data/scadabridge-secrets.db, so it needs the same + # per-node volume the site nodes already have. Without it the store sits in the + # image's writable layer and is destroyed by every container recreate. + - ./central-node-b/data:/app/data - ./central-node-b/logs:/app/logs networks: - scadabridge-net @@ -103,6 +161,9 @@ services: stop_grace_period: 30s container_name: scadabridge-site-a-a environment: + # Follower half of the pull-only gRPC secrets hub (anchors at the top of this + # file). site-b and site-c deliberately carry neither. + <<: [*secrets-hub-env, *secrets-hub-site-env] SCADABRIDGE_CONFIG: Site ports: - "9021:8082" # Akka remoting (host access for debugging) @@ -123,6 +184,9 @@ services: stop_grace_period: 30s container_name: scadabridge-site-a-b environment: + # Follower half of the pull-only gRPC secrets hub (anchors at the top of this + # file). site-b and site-c deliberately carry neither. + <<: [*secrets-hub-env, *secrets-hub-site-env] SCADABRIDGE_CONFIG: Site ports: - "9022:8082" # Akka remoting diff --git a/docs/plans/2026-08-07-secrets-grpc-hub-live-gate.md b/docs/plans/2026-08-07-secrets-grpc-hub-live-gate.md new file mode 100644 index 00000000..f4f33c4b --- /dev/null +++ b/docs/plans/2026-08-07-secrets-grpc-hub-live-gate.md @@ -0,0 +1,339 @@ +# Secrets gRPC hub — live gate on the docker cluster + +**Result: 3 of 4 checks PASS. Check 4 FAILS one clause of its stated criterion.** +**NOT MERGED** — the gate's merge condition is 4/4. Run 2026-08-07 on the local 8-node +docker cluster. + +## Purpose + +Terminal gate (Task 7) for `feat/secrets-grpc-hub`, which wires +`ZB.MOM.WW.Secrets.Replicator.Grpc` 0.4.0 into ScadaBridge: the central role hosts a +**pull-only** `secrets_hub.v1` gRPC hub on its existing h2c control-plane listener, and a +site node runs a sweep that converges its **own local SQLite store** from that hub. + +This is the ScadaBridge half of the production-topology decision in +[scadaproj#3](https://gitea.dohertylan.com/dohertj2/scadaproj/issues/3) (design: +`scadaproj/docs/plans/2026-08-07-secrets-production-topology-and-grpc-hub.md`). The +SqlServer replicator is not the production path here: it requires every site node to hold a +connection string to central's database, which breaks ScadaBridge's standing rule that +sites talk to central, not to central's DB. + +**Scope: rig config only.** No product code was changed by this gate — see +[Defects](#defects-found). + +## Rig state + +| Fact | Value | +|---|---| +| Repo / branch | `ScadaBridge` @ `feat/secrets-grpc-hub` | +| Git SHA gated | `fc784b413713b1643a21d573e55c00bcc05bd8ee` (`fc784b41`) | +| Image | `scadabridge:latest` = `2958aa2c2bbb`, built 2026-08-07 11:39 UTC | +| `ZB.MOM.WW.ScadaBridge.Host.dll` | `09cf76dc550d32fe…` — **byte-identical on all 8 nodes** | +| Secrets libs | `ZB.MOM.WW.Secrets*` **0.4.0** (`Directory.Packages.props`) | +| Working tree at build | only `docker/docker-compose.yml` modified (rig config, not in the image) + untracked `.claude/` | +| Hub enabled on | `central-a`, `central-b` (host) and `site-a-a`, `site-a-b` (followers) | +| Hub NOT enabled on | `site-b-a`, `site-b-b`, `site-c-a`, `site-c-b` — no `Secrets__*` env, no store file, no hub/sweep log line (verified) | +| Sweep interval | `00:00:30` — the **product default**, deliberately not shortened, so the timings below are real | +| KEK | one shared committed **dev-only** key, identical on all four (`kek_id` sha256 `7451bcbc1f1f` on every row observed) | + +`site-b` and `site-c` were left off on purpose: the default-OFF posture is proven side by +side on one rig, the same way `site-a` is the rig's only LocalDb-replicated pair. + +### Rig-config change made by this gate + +`docker/docker-compose.yml` gained two YAML anchors and applied them to four services: + +- `x-secrets-hub-env` (central pair + site-a pair) — dev KEK, `Secrets__SqlitePath`, + `Secrets__Replication__Enabled=true`, `Secrets__Replication__Mode=Grpc`, + `Secrets__GrpcHub__BearerToken`. +- `x-secrets-hub-site-env` (site-a pair only) — + `Secrets__GrpcHub__Endpoint=http://scadabridge-central-a:8083`. + +All values are DEV-ONLY and committed under the same exception the mesh PSKs and +`InboundApi:ApiKeyPepper` already use. `Secrets__SqlitePath` points at `/app/data` because +the appsettings default is a relative path resolving to `/app` — inside the image's +writable layer, destroyed by every container recreate. The **central pair gained a +`./central-node-*/data:/app/data` volume** for this; the site pairs already had one. + +## Method + +Secrets were seeded, read and deleted with the `ZB.MOM.WW.Secrets.Cli` (`secret`) published +from `scadaproj/ZB.MOM.WW.Secrets` (0.4.0) and run **in a throwaway container** mounting one +node's `data` volume, with that node's KEK. This is the method the sibling OtOpcUa gate used +the same day (`OtOpcUa/docs/plans/2026-08-07-secrets-pair-local-live-gate.md`). + +The CentralUI `/admin/secrets` page was **not** used, and for this topology that costs +nothing: the hub is a pure reader of central's store on every call, with no write-triggered +path and no manifest cache. A row written by any process into central's store is on exactly +the same path a row written through the page would be. (Contrast the Akka replicator, where +the two differ — hence that gate's residual 1, which does not apply here.) + +Ciphertext identity is evidenced as a **SHA-256 over the encrypted columns only** +(`ciphertext‖nonce‖tag‖wrapped_dek‖wrap_nonce‖wrap_tag`), plus a hash of `kek_id`. No +plaintext, no token and no key material appears in this document. The secret values used +were throwaway smoke strings. + +gRPC calls for check 4 were made with `fullstorydev/grpcurl` on the `scadabridge-net` +network, against the committed `secrets_hub.proto`. + +## Results + +| # | Check | Result | Timing | +|---|---|---|---| +| 1 | Convergence — central write reaches both site-a nodes, byte-identical, decrypt-verified | **PASS** | **5 s** (budget 30 s + margin) | +| 2 | Cold-boot offline — site pair boots and serves last-known-good with central down; convergence resumes | **PASS** | resumed in **14 s** / **43 s** | +| 3 | Tombstone — delete propagates, survives a pair restart | **PASS** | **≤ 9 s** | +| 4 | Auth negatives + log hygiene | **FAIL** (1 clause of 3) | — | + +Baseline before seeding: all four stores empty. + +### Check 1 — convergence · PASS + +Seeded `hub-gate-smoke-1` on **`central-a` only** at 11:46:57Z. + +``` +central-node-a hub-gate-smoke-1 rev=0 del=0 cipher_sha256=00a14a2eb8de987fb75a4515 kek=7451bcbc1f1f updated=2026-08-07T11:46:57.1403922+00:00 +site-a-node-a hub-gate-smoke-1 rev=0 del=0 cipher_sha256=00a14a2eb8de987fb75a4515 kek=7451bcbc1f1f updated=2026-08-07T11:46:57.1403922+00:00 +site-a-node-b hub-gate-smoke-1 rev=0 del=0 cipher_sha256=00a14a2eb8de987fb75a4515 kek=7451bcbc1f1f updated=2026-08-07T11:46:57.1403922+00:00 +``` + +Ciphertext hash, `kek_id`, `revision` and `updated_utc` are **identical on all three** — the +row is relayed verbatim, not re-encrypted, so the last-writer-wins ordering key survives the +hop. First poll after the seed (11:47:05Z) already showed both followers converged. + +Both halves logged it, and central served the fetch: + +``` +site-a-a [11:47:02 INF] Secret hub sync converged: pulled 1 row(s) from the central hub. +site-a-b [11:47:02 INF] Secret hub sync converged: pulled 1 row(s) from the central hub. +central-a [11:47:02 INF] Request finished HTTP/2 POST .../SecretsHub/GetSecrets - 200 application/grpc 0.9899ms +``` + +**Convergence = 11:46:57Z → 11:47:02Z = 5 s**, one sweep tick. + +**Decrypt-verified on both followers**: `secret get hub-gate-smoke-1` returned the exact +plaintext seeded on central-a, on `site-a-a` and `site-a-b`. + +### Check 2 — cold-boot offline · PASS + +Central pair stopped 11:48:27Z → 11:48:37Z. + +**Sweep degrades, does not crash.** One warning per interval on each follower, carrying the +transport fault and nothing else: + +``` +site-a-a [11:49:02 WRN] Secret hub sync failed; the node continues serving its local store and will retry on the next interval. + Grpc.Core.RpcException: Status(StatusCode="Unavailable", Detail="Error connecting to subchannel." …) +``` + +**Site pair restarted together** (the supported pattern) 11:49:31Z → 11:50:12Z, with central +still down. `site-a-a` booted at 11:49:42Z; its `SyncOnStartup` sweep warned at 11:49:42Z and +retried at 11:50:12 / 11:50:42 / 11:51:12 / 11:51:42 / 11:52:12 / 11:52:42. Both containers +`running`, `RestartCount=0`, **zero** `FATAL` / `Unhandled exception` / +`Application startup exception` lines. + +**Last-known-good served with central down** — at 11:51:08Z, `secret get hub-gate-smoke-1` +returned the correct plaintext on **both** `site-a-a` and `site-a-b`. + +Central pair started 11:51:37Z. `hub-gate-smoke-2` seeded on central-a at 11:52:29Z — **after +the site nodes were already up**, so this proves convergence *resumed* rather than +re-bootstrapped: + +``` +site-a-b [11:52:43 INF] Secret hub sync converged: pulled 1 row(s) from the central hub. → 14 s +site-a-a [11:53:12 INF] Secret hub sync converged: pulled 1 row(s) from the central hub. → 43 s +``` + +Both rows byte-identical to central's (`cipher_sha256=4f022e222cfc2f8f77dcfb0b`, +`updated=2026-08-07T11:52:29.6983529+00:00`). + +**Honest note on the 43 s.** `site-a-a`'s first post-outage sweep (11:52:42Z) *still* failed, +with `SocketException: Name or service not known` — a stale DNS/subchannel view of a +container that had been `docker stop`ped and `docker start`ed. It recovered unaided on the +very next tick. So after a central outage, worst-case convergence on this rig is **two** +sweep intervals, not one. Bounded, self-healing, no intervention — but recorded so the 5 s +figure from check 1 is not read as the post-outage number. + +### Check 3 — tombstone · PASS + +`hub-gate-smoke-1` deleted on central-a at 11:54:05Z. `hub-gate-smoke-2` left live as a +control. + +``` +central-node-a hub-gate-smoke-1 rev=1 del=1 updated=2026-08-07T11:54:05.9812744+00:00 deleted=2026-08-07T11:54:05.9812744+00:00 +site-a-node-a hub-gate-smoke-1 rev=1 del=1 updated=2026-08-07T11:54:05.9812744+00:00 deleted=2026-08-07T11:54:05.9812744+00:00 +site-a-node-b hub-gate-smoke-1 rev=1 del=1 updated=2026-08-07T11:54:05.9812744+00:00 deleted=2026-08-07T11:54:05.9812744+00:00 +``` + +Both followers held the tombstone at the 11:54:14Z poll — **≤ 9 s**. Delete is a soft +tombstone (`is_deleted=1`, `revision` bumped, `updated_utc` restamped), which is what lets it +win under last-writer-wins. `secret get` returned `{"error":"not-found"}` and `secret list` +returned only `hub-gate-smoke-2` on both followers. + +**No resurrection across a pair restart.** Site pair restarted together 11:55:09Z → +11:55:50Z, **with central up and sweeping the whole time** — so the sweep had every +opportunity to re-pull the row. Re-checked at 11:57:05Z (75 s / 2+ sweeps later): + +- `hub-gate-smoke-1` still `rev=1 del=1` on both, `updated_utc` unchanged; + `secret get` → `{"error":"not-found"}` on both. +- `hub-gate-smoke-2` survived on both and **decrypted to the correct plaintext** on both. + +### Check 4 — auth negatives + log hygiene · FAIL (one clause) + +Three calls to `SecretsHub/GetManifest` on `scadabridge-central-a:8083` at 11:57:31–32Z: + +| Call | Result | +|---|---| +| **no** `authorization` header | `Code: Unauthenticated` / `Message: Unauthenticated.` | +| **wrong** bearer (`Bearer not-the-hub-token`) | `Code: Unauthenticated` / `Message: Unauthenticated.` | +| correct bearer (positive control) | `200`, manifest returned (2 entries) | + +**PASS — identical denial.** Both negatives are byte-identical in code and detail: a caller +learns it was refused and nothing else. The positive control proves the endpoint is live and +the refusals are the auth gate, not a dead route. + +**PASS — log hygiene, zero hits.** Every one of the **8** nodes' `docker logs` *and* every +on-disk Serilog file under `docker/*/logs/` was grepped for the dev bearer token value, the +dev KEK value, and both secret plaintexts: + +``` +central-a token=0 kek=0 plain1=0 plain2=0 authz-header-mentions=0 +central-b token=0 kek=0 plain1=0 plain2=0 authz-header-mentions=0 +site-a-a token=0 kek=0 plain1=0 plain2=0 authz-header-mentions=0 +site-a-b token=0 kek=0 plain1=0 plain2=0 authz-header-mentions=0 +site-b-a / site-b-b / site-c-a / site-c-b all zero +on-disk log files: zero matching files for all four search strings +``` + +The token appears only where it legitimately may: the committed rig config +(`docker/docker-compose.yml`) and the resulting container environment. + +**FAIL — no server-side *warning*.** The criterion asks for a server-side warning on denial. +What the hub actually emits is one **Information**-level line per denial, from +`Grpc.AspNetCore.Server`, not from the hub's own gate: + +``` +[11:57:31 INF] Error status code 'Unauthenticated' with detail 'Unauthenticated.' raised. +[11:57:32 INF] Error status code 'Unauthenticated' with detail 'Unauthenticated.' raised. +``` + +Nothing at `WRN` or above was emitted by either denial — verified by dumping **every** log +line in the 11:57:31–32Z window, not by a filtered grep. + +This is **not a wiring mistake and not fixable in this repo**: `SecretsHubAuthInterceptor` +(in `ZB.MOM.WW.Secrets.Replicator.Grpc` 0.4.0) deliberately logs *nothing* on a denial. Its +only warning is `WarnUnconfiguredOnce`, emitted once when the hub has **no token configured +at all** — which is not the case here. So the shipped design and this gate's criterion +disagree, and the disagreement lives in the library. + +The consequence is real but narrow: a follower whose token was mis-rotated is refused on +every sweep and **stops converging**, while central records that only at Information. An +operator watching for warnings sees the follower's own `Secret hub sync failed` WRN (which +does fire), but central shows nothing above INF. + +**Not patched in ScadaBridge.** A host-side interceptor could log the denial, but that would +(a) contradict a decision the library documents and reasons about explicitly, in the +consumer rather than at the layer that owns it, and (b) put an unbounded log write on an +unauthenticated endpoint. Both are worse than the gap. The correct fix is a rate-limited +warning in `SecretsHubAuthInterceptor`, or an explicit decision that Information is right +and this criterion should be relaxed. That call is not this gate's to make — see +[Follow-ups](#follow-ups). + +## Defects found + +**None in product code.** One defect in the *gate method*, caught and corrected before any +result was recorded: + +**Host-side seeding of a bind-mounted SQLite store is not coherent with the running +container** (Docker Desktop for macOS). The first attempt at check 1 seeded central-a's +store with the CLI running natively on the host. The row was visible to the host and to a +fresh helper container, but the **running node never saw it**: four consecutive sweeps +returned an empty manifest, no `GetSecrets` was ever issued, and a `wal_checkpoint(TRUNCATE)` +did not help. Restarting central-a then **lost the row entirely** — it was gone from both the +host and container views. Cause is the virtiofs/page-cache boundary between the macOS host +and the Linux guest: SQLite's WAL and `-shm` mappings are not shared across it. + +Corrected by doing **every** store access — seed, read, delete — from a throwaway container +on the same guest kernel, which is what the sibling OtOpcUa gate already did. The rig was +returned to an empty-store baseline before check 1 was re-run. Recorded because the failure +mode is a convincing false negative: it looks exactly like "the hub is broken". + +## Residuals + +1. **The hub client dials a SINGLE endpoint — no failover.** Known and documented on the + branch. Observed live and directly contrasted on the same node in the same minute: + + ``` + site-a-a [11:51:13 WRN] Central control-plane endpoint http://scadabridge-central-a:8083 is + unavailable; site site-a failed over to http://scadabridge-central-b:8083. + site-a-a [11:51:12 WRN] Secret hub sync failed; the node continues serving its local store … + ``` + + `ScadaBridge:Communication:CentralGrpcEndpoints` is a list and failed over; the hub client + stalled. Survivable — the sweep is best-effort and the node keeps its full local + last-known-good store — but secrets stop converging until that one central node returns. + +2. **The central pair does NOT converge with itself, and this makes residual 1 sharper than + it reads.** Both central nodes host the hub (verified: `central-b` answers, and refuses an + unauthenticated call identically). But `central-b`'s store stayed **empty** for the whole + run while `central-a` held both secrets: + + ``` + central-b, correct bearer, 11:59:32Z: {} ← empty manifest, authenticated + central-node-b (store): (empty) + ``` + + Nothing replicates central-a ↔ central-b in `Mode=Grpc` — the topology is pull-only + *central → sites*. So a hypothetical failover to `central-b` would not merely stall, it + would succeed against an empty hub. By the reconciler's pull-only algorithm that is not + data loss (a name absent from the manifest is simply not pulled, never deleted), but it + *is* a silent stop to convergence, and a deployment that authors a secret on the central + node the sites do **not** dial will see it reach nobody. **Not live-tested here** — + asserted from the algorithm plus the observed empty manifest. Any production enablement + needs an answer for how the central pair itself converges. + +3. **Both site-a nodes dial the same central node.** Not a rig shortcut — it is the only + shape the client supports (residual 1). Worth restating so the run is not read as having + exercised two independent hub sources. + +4. **The `/admin/secrets` write path was not exercised.** Seeding was CLI-into-central's-store. + For this topology that is the same path (see [Method](#method)), so it is a coverage note, + not a gap in the result — but the page's own Blazor/authz behaviour is unproven by this + gate. `ScadaBridge#22` covered it separately. + +5. **Delete-while-a-follower-is-offline was not tested.** Check 3 restarts the pair *after* + both already held the tombstone. The harder resurrection case — deleting at central while + a follower is down, then bringing it back holding a live copy — is covered by the + library's last-writer-wins tests but not live here. Under a pull-only hub it is + structurally safer than in a bidirectional topology (the follower can never push its stale + live row back), which is why it was not prioritised. + +6. **Replication stays default-OFF in the product.** `Secrets:Replication:Enabled` is `false` + and `Mode` is `SqlServer` in the shipped `appsettings.json`. It is enabled on this docker + rig only, on four of eight nodes, with a dev KEK and a dev token. Production enablement + additionally needs a real KEK supplied out of band and a real token — never committed, and + never a `${secret:}` reference. + +## Follow-ups + +- **File against `ZB.MOM.WW.Secrets.Replicator.Grpc`:** decide whether a denied hub call + should emit a rate-limited `Warning` on the server. Today it emits nothing from the gate + itself, so a mis-rotated follower token is visible at central only at Information level. + This is the sole reason this gate is not 4/4. +- **File against ScadaBridge:** endpoint failover for the hub client, and/or a documented + answer for how the central pair's own stores converge (residual 2). Both are the same + underlying question — "which central node is authoritative for secrets". + +## Conclusion + +The pull-only gRPC secrets hub **works, and works the way the branch says it does**: +convergence is one sweep tick, ciphertext is relayed verbatim so last-writer-wins ordering +survives the hop, a site node boots and serves its full last-known-good store with the entire +central pair stopped, convergence resumes unaided when central returns, tombstones propagate +and do not resurrect across a pair restart, the auth gate refuses uniformly, and no token, +KEK or plaintext reaches any log on any of the eight nodes. + +One clause of check 4 is not met — denials are recorded at Information, not Warning — and +that clause is a property of the library, not of this branch. The gate's merge condition is +4/4, so **this branch is not merged by this gate**. Resolving the logging question (fix the +library, or relax the criterion) is the only thing standing between this result and a merge.