Compare commits
26 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Joseph Doherty
|
06f2df4f89 | ||
|
Joseph Doherty
|
d9f5fbb664 | ||
|
Joseph Doherty
|
f48a748f37 | ||
|
Joseph Doherty
|
fd22f5ce0a | ||
|
Joseph Doherty
|
6538216b0c | ||
|
Joseph Doherty
|
99254b71de | ||
|
Joseph Doherty
|
eb59c4244f | ||
|
Joseph Doherty
|
96192950a0 | ||
|
Joseph Doherty
|
ec2aa2bbac | ||
|
Joseph Doherty
|
e5503857df | ||
|
Joseph Doherty
|
7c7989d176 | ||
|
Joseph Doherty
|
556febd86f | ||
|
Joseph Doherty
|
3e64959582 | ||
|
Joseph Doherty
|
5c2db9fe70 | ||
|
Joseph Doherty
|
631ce5bfce | ||
|
Joseph Doherty
|
3955cb4f28 | ||
|
Joseph Doherty
|
caaa855362 | ||
|
Joseph Doherty
|
7a085444e4 | ||
|
Joseph Doherty
|
298a9af59e | ||
|
Joseph Doherty
|
10f752df02 | ||
|
Joseph Doherty
|
25f768f379 | ||
|
Joseph Doherty
|
381d26d1b1 | ||
|
Joseph Doherty
|
a61865daa0 | ||
|
Joseph Doherty
|
8cb512ac51 | ||
|
Joseph Doherty
|
290acfb1f0 | ||
|
Joseph Doherty
|
81cb455f19 |
@@ -42,7 +42,8 @@
|
||||
"LifecycleTimeout": "00:00:30",
|
||||
"QueryTimeout": "00:00:30",
|
||||
"TransportHeartbeatInterval": "00:00:05",
|
||||
"TransportFailureThreshold": "00:00:15"
|
||||
"TransportFailureThreshold": "00:00:15",
|
||||
"CentralFetchBaseUrl": "http://scadabridge-env2-traefik"
|
||||
},
|
||||
"HealthMonitoring": {
|
||||
"ReportInterval": "00:00:30",
|
||||
|
||||
@@ -42,7 +42,8 @@
|
||||
"LifecycleTimeout": "00:00:30",
|
||||
"QueryTimeout": "00:00:30",
|
||||
"TransportHeartbeatInterval": "00:00:05",
|
||||
"TransportFailureThreshold": "00:00:15"
|
||||
"TransportFailureThreshold": "00:00:15",
|
||||
"CentralFetchBaseUrl": "http://scadabridge-env2-traefik"
|
||||
},
|
||||
"HealthMonitoring": {
|
||||
"ReportInterval": "00:00:30",
|
||||
|
||||
@@ -45,7 +45,8 @@
|
||||
"LifecycleTimeout": "00:00:30",
|
||||
"QueryTimeout": "00:00:30",
|
||||
"TransportHeartbeatInterval": "00:00:05",
|
||||
"TransportFailureThreshold": "00:00:15"
|
||||
"TransportFailureThreshold": "00:00:15",
|
||||
"CentralFetchBaseUrl": "http://scadabridge-traefik"
|
||||
},
|
||||
"HealthMonitoring": {
|
||||
"ReportInterval": "00:00:30",
|
||||
|
||||
@@ -45,7 +45,8 @@
|
||||
"LifecycleTimeout": "00:00:30",
|
||||
"QueryTimeout": "00:00:30",
|
||||
"TransportHeartbeatInterval": "00:00:05",
|
||||
"TransportFailureThreshold": "00:00:15"
|
||||
"TransportFailureThreshold": "00:00:15",
|
||||
"CentralFetchBaseUrl": "http://scadabridge-traefik"
|
||||
},
|
||||
"HealthMonitoring": {
|
||||
"ReportInterval": "00:00:30",
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
# Deploy hangs (120 s) when the flattened instance config exceeds Akka's default frame size
|
||||
|
||||
**Date:** 2026-06-26 · **Status:** RESOLVED (2026-06-26) · **Severity:** High (silently un-deployable instances) · **Area:** Deployment Manager / Cluster Communication
|
||||
|
||||
## Resolution (2026-06-26)
|
||||
|
||||
Fixed via the **notify-and-fetch** rework (the primary recommendation below), not the frame-size stopgap. The full flattened config no longer travels inside any Akka message: central stages it in a `PendingDeployment` row and sends a small `RefreshDeploymentCommand`; the site fetches the config over HTTP using a per-deployment token. Both frame-trapped Akka hops are fixed — the central→site deploy hop **and** the site active→standby `SiteReplicationActor` hop (which carried the same config across an intra-site Akka hop with the same silent-drop trap). The secondary `AskTimeoutException` mis-classification (below) is fixed, and a startup `SiteReconciliationActor` self-heal plus a topology-page performance fix shipped alongside.
|
||||
|
||||
- **Design:** [`docs/plans/2026-06-26-deploy-config-notify-and-fetch-design.md`](../plans/2026-06-26-deploy-config-notify-and-fetch-design.md)
|
||||
- **Plan:** [`docs/plans/2026-06-26-deploy-config-notify-and-fetch.md`](../plans/2026-06-26-deploy-config-notify-and-fetch.md)
|
||||
- **Validated:** live docker-cluster smoke — a previously-hanging deploy now completes in ~0.11 s; reconciliation heals single-node and concurrent-both-missing gaps.
|
||||
|
||||
The diagnosis below is retained as the historical record of how the bug was found and reasoned about.
|
||||
|
||||
## Summary
|
||||
|
||||
Deploying an instance whose **flattened configuration JSON is large enough** that the
|
||||
`DeployInstanceCommand` Akka message exceeds Akka.Remote's default `maximum-frame-size`
|
||||
(**128 KB / `128000b`**) causes the message to be **silently dropped in transit** between the
|
||||
central and site nodes. The central's deploy `Ask` then times out after exactly **120 s**
|
||||
(`Communication:DeploymentTimeout`), the deployment is recorded as failed, and **the instance
|
||||
cannot be (re)deployed at all** until the config shrinks back under the limit.
|
||||
|
||||
It was discovered while adding a **third composition of the same base template** (a generic
|
||||
`DelmiaReceiver` added to `CvdReactor`, which already composes `LeftDelmiaReceiver` +
|
||||
`RightDelmiaReceiver`). Each composition fully expands the composed template's members into the
|
||||
flattened config, so the 3rd copy tipped the message over 128 KB.
|
||||
|
||||
## How it manifests
|
||||
|
||||
- CLI/UI deploy returns `{"error":"Deployment error: Timeout after 120.00 seconds","code":"COMMAND_FAILED"}` (or a 504 via the management API) after ~120 s.
|
||||
- `deploy status` shows the record going `InProgress (1) → Failed (3)` with `errorMessage = "Deployment error: Timeout after 120.00 seconds"`.
|
||||
- The **running instance is unaffected** — deploys are atomic, so it keeps serving its last successful deployment.
|
||||
- Smaller configs deploy normally and fast (the working baseline here, the 2-receiver side-based config, completed in **0.04 s**).
|
||||
|
||||
## Root cause
|
||||
|
||||
The deploy ships the **entire flattened config inline** as a JSON string inside the Akka message,
|
||||
and Akka.Remote's frame size is left at the un-configured default:
|
||||
|
||||
1. **Full config carried in the message** — `DeploymentService.cs` serializes the flattened config
|
||||
(`JsonSerializer.Serialize(flattenedConfig)`) and puts it in
|
||||
`DeployInstanceCommand.FlattenedConfigurationJson`
|
||||
(`src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Deployment/DeployInstanceCommand.cs`).
|
||||
2. **Sent central → site over Akka** — via ClusterClient
|
||||
(`CentralCommunicationActor` → `ClusterClient.Send("/user/site-communication", …)`).
|
||||
3. **No frame-size override** — `AkkaHostedService.BuildHocon`
|
||||
(`src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs`, ~lines 240-244) sets only
|
||||
`dot-netty.tcp { hostname; port }`. There is **no `maximum-frame-size`** and **no appsettings
|
||||
knob** for it, so the Akka.Remote 1.5.62 default of `128000b` applies (and
|
||||
`log-frame-size-exceeding` defaults to `off`, so there isn't even a warning).
|
||||
4. **Amplifier** — no custom Akka serializer / `serialization-bindings` is configured, so the
|
||||
already-serialized `configJson` is JSON-escaped **again** inside the envelope by the default
|
||||
serializer (every `"` → `\"`), roughly doubling that portion of the on-wire payload. So the raw
|
||||
flattened JSON only needs to be ~60-70 KB to blow the 128 KB frame.
|
||||
|
||||
When the encoded frame exceeds the limit, the transport drops **that one message** without tearing
|
||||
down the association (heartbeats/telemetry keep flowing, so the site still reports healthy), the
|
||||
site never receives the deploy, and the central's `Ask` (timeout = `DeploymentTimeout` = 2 min)
|
||||
expires.
|
||||
|
||||
### Why the 3rd same-base composition is the trigger
|
||||
|
||||
The flattener (`FlatteningService`) fully expands **each** composition's attributes/alarms/scripts,
|
||||
path-qualified — so N compositions of the same base ≈ N× those members in the config. The config
|
||||
grows roughly linearly with composition count; here **2 `DelmiaReceiver` copies sat under the limit,
|
||||
the 3rd pushed it over**. Removing the composed *scripts* did not help, because the *attribute* set
|
||||
alone (×3) already exceeded the frame.
|
||||
|
||||
## Evidence (live logs, wonder-app-vd03, 2026-06-26)
|
||||
|
||||
Central log:
|
||||
```
|
||||
07:44:28.033 [INF] Sending deployment 82a39ef5… for instance Z28061Sim to site site-a
|
||||
… (no further deploy activity; only DB/health/telemetry) …
|
||||
07:46:28.038 [ERR] Deployment 82a39ef5… for instance Z28061Sim failed
|
||||
07:46:28.039 [ERR] Management command MgmtDeployInstanceCommand failed
|
||||
07:46:28.039 [WRN] Dead letter: ManagementError from akka://…/temp/3d to deadLetters
|
||||
```
|
||||
Site log (same window): **nothing** — no deploy receipt, no error. The last deploy the site ever
|
||||
received was the smaller side-based config:
|
||||
```
|
||||
05:51:46.429 [INF] Deploying instance Z28061Sim, deploymentId=e531d216… ← worked
|
||||
```
|
||||
No `OversizedPayloadException`, serialization error, or deserialization error appears on **either**
|
||||
node — consistent with a silent transport-level frame drop. The differentiator is purely config size
|
||||
(2-receiver config delivered & applied; 3-receiver config never arrived).
|
||||
|
||||
## Secondary bug (found during the same investigation)
|
||||
|
||||
`AskTimeoutException` does **not** derive from `System.TimeoutException`, so the catch in
|
||||
`DeploymentService.cs` (~line 312) mis-classifies a genuine site Ask-timeout as a generic
|
||||
`"Deployment error: …"` instead of the `"Communication failure: …"` / timeout branch. Consequence:
|
||||
the DeploymentManager-006 "query-site-before-redeploy" reconciliation (keyed off the
|
||||
`Communication failure:` prefix) does **not** trigger after these failures. Fix the classification so
|
||||
`AskTimeoutException`/`OperationCanceledException` are treated as timeouts.
|
||||
|
||||
## Recommended fix
|
||||
|
||||
> **Shipped:** the **Primary** option below (notify-and-fetch) was implemented and merged on 2026-06-26 — see the Resolution section at the top. The stopgap frame-size bump and the config-only workaround were NOT taken.
|
||||
|
||||
### Primary (recommended): notify-and-fetch instead of push
|
||||
|
||||
Stop shipping the full flattened config inside the Akka message. Instead:
|
||||
|
||||
1. Central persists the pending flattened config (it already persists `DeployedConfigSnapshots` on
|
||||
success — extend to a pending/by-`deploymentId` store).
|
||||
2. Central sends the site a **small** "apply deployment `<deploymentId>` for instance `<id>`
|
||||
(revisionHash `<hash>`)" notification.
|
||||
3. The **site pulls** the config by `deploymentId` and applies it, then sends a completion/status
|
||||
response.
|
||||
|
||||
This decouples the Akka message size from the config size entirely (no instance is ever
|
||||
un-deployable due to size), and it fits the **existing pull-based gRPC telemetry pattern** the site
|
||||
already uses (`PullSiteCalls`, `PullAuditEvents` over `SiteStreamService`). A new
|
||||
`PullDeploymentConfig`/`SiteStreamService` RPC (or reuse of the management API) is the natural home.
|
||||
Keep the completion ack so the central still gets authoritative success/failure.
|
||||
|
||||
### Stopgap (until the above ships)
|
||||
|
||||
Add an explicit frame size + matching buffers to the `dot-netty.tcp` block in
|
||||
`AkkaHostedService.BuildHocon`, e.g.:
|
||||
```hocon
|
||||
remote {
|
||||
dot-netty.tcp {
|
||||
hostname = …
|
||||
port = …
|
||||
maximum-frame-size = 4000000b # 4 MB
|
||||
send-buffer-size = 8000000b
|
||||
receive-buffer-size = 8000000b
|
||||
log-frame-size-exceeding = 1000000b # warn before silently dropping
|
||||
}
|
||||
}
|
||||
```
|
||||
This is a **code** change (no appsettings override exists) and must be deployed to **both** central
|
||||
and site (they share the Host); rebuild + redeploy per `deploy/wonder-app-vd03/RUNBOOK.md`
|
||||
(Upgrading). Consider exposing it via `appsettings`/`CommunicationOptions` so it's tunable without a
|
||||
rebuild. Also fix the `AskTimeoutException` classification.
|
||||
|
||||
### Config-only workaround (specific to this case)
|
||||
|
||||
For the generic-`DelmiaReceiver` rollout specifically: removing the `Left`/`Right` `DelmiaReceiver`
|
||||
compositions and keeping **only** the single generic `DelmiaReceiver` makes the config *smaller* than
|
||||
the working 2-receiver baseline, so it deploys under the current limit — and it's the intended
|
||||
"no sides" end state anyway (each machine bound to its own generic Galaxy receiver,
|
||||
`DelmiaReceiver_037/038/039`). This unblocks without an engine change, but does not fix the
|
||||
underlying fragility for any other instance that legitimately needs a large config.
|
||||
|
||||
## Current operational state (to clean up)
|
||||
|
||||
> **Engine fix resolves the blocker.** With notify-and-fetch merged, a large flattened config is no longer size-capped, so the generic-`DelmiaReceiver` rollout (and the `Download` handshake re-add) can proceed on `Z28061Sim` without hitting this bug. The items below are the operational re-rollout steps, now unblocked — they are NOT engine work.
|
||||
|
||||
- `Z28061Sim` is running on deployment **id 55** (side-based, with the earlier `side`-crash fix) and
|
||||
is healthy, but is **not currently redeployable**: `CvdReactor` has the extra generic
|
||||
`DelmiaRecv`→`DelmiaReceiver_037` composition + binding, so every redeploy hits this bug.
|
||||
- To restore redeployability, either apply the config-only workaround above (consolidate to one
|
||||
generic receiver) or roll back the generic composition + restore the side-based
|
||||
`ProcessRecipeDownload`.
|
||||
- The `Download` handshake method (mirroring `MESReceiver.MoveIn`) is currently removed from the
|
||||
`DelmiaReceiver` base template (template 12); re-add it once a deployable path is chosen.
|
||||
|
||||
## Files
|
||||
|
||||
- `src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs` (BuildHocon — missing `maximum-frame-size`)
|
||||
- `src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentService.cs` (inline config serialize; AskTimeout mis-classification)
|
||||
- `src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Deployment/DeployInstanceCommand.cs` (`FlattenedConfigurationJson`)
|
||||
- `src/ZB.MOM.WW.ScadaBridge.Communication/Actors/CentralCommunicationActor.cs` (ClusterClient send)
|
||||
- `src/ZB.MOM.WW.ScadaBridge.Communication/CommunicationOptions.cs` (`DeploymentTimeout = 2 min`)
|
||||
- `src/ZB.MOM.WW.ScadaBridge.TemplateEngine/Flattening/FlatteningService.cs` (per-composition member expansion)
|
||||
</content>
|
||||
@@ -0,0 +1,195 @@
|
||||
# Deploy + Replication Config Transport — Notify-and-Fetch Design
|
||||
|
||||
**Date:** 2026-06-26 · **Status:** APPROVED (design) · **Area:** Deployment Manager / Site Runtime / Cluster Communication
|
||||
**Fixes:** [`docs/known-issues/2026-06-26-deploy-config-exceeds-akka-frame-size.md`](../known-issues/2026-06-26-deploy-config-exceeds-akka-frame-size.md)
|
||||
|
||||
## 1. Problem
|
||||
|
||||
The flattened instance config travels over Akka on **two** hops today, and both are bounded by Akka.Remote's default `maximum-frame-size` (**128 KB / `128000b`**). A large config (e.g. a 3rd composition of the same base template) silently breaks both:
|
||||
|
||||
1. **Central → site (deploy).** `DeploymentService` serializes the whole flattened config into `DeployInstanceCommand.FlattenedConfigurationJson` and Asks the site over ClusterClient. An oversized frame is **silently dropped**; the central `Ask` times out after 120 s (`Communication:DeploymentTimeout`); the instance becomes un-deployable until the config shrinks.
|
||||
|
||||
2. **Site active → standby (replication).** After a successful apply, `DeploymentManagerActor` does `_replicationActor.Tell(new ReplicateConfigDeploy(instanceName, command.FlattenedConfigurationJson, …))`; `SiteReplicationActor` relays it node-to-node via `ActorSelection(RootActorPath(peer)/"user"/"site-replication").Tell(…)`. This is **fire-and-forget** — an oversized `ApplyConfigDeploy` is silently dropped with *no timeout and no error*. The deploy "succeeds," the active node runs fine, and the standby silently lacks the instance — surfacing only at **failover**, when the now-active node recovers from its SQLite and the instance simply isn't there.
|
||||
|
||||
The JSON is double-escaped inside the Akka envelope (default serializer), so ~60–70 KB of raw config is enough to blow the 128 KB frame. `log-frame-size-exceeding` defaults off, so there isn't even a warning.
|
||||
|
||||
## 2. Goal & guiding principle
|
||||
|
||||
Eliminate the frame-size failure on **both** hops, permanently and at any config size, **without** raising `maximum-frame-size`.
|
||||
|
||||
**Principle:** *small control messages stay on Akka; the bulk config never touches Akka.* The config always moves over **HTTP from central** (the single source of truth) to whichever node needs it — the active node (which applies + runs it) and the standby node (which persists it for future recovery).
|
||||
|
||||
This mirrors how the site already works at startup: a site recovers its instances **purely from local SQLite** (`deployed_configurations`), never contacting central. A deploy thus becomes an *on-demand, single-instance version of startup recovery*: "your definition changed — fetch it and refresh." Only the config **source** changes; the apply path is unchanged.
|
||||
|
||||
### Why notify-and-fetch over alternatives
|
||||
|
||||
- **Push over gRPC (central→site):** would need host-aware routing (the `DeploymentManager` singleton runs only on the active node; the gRPC server runs on both), and still leaves the *intra-site* replication hop broken.
|
||||
- **Raise Akka `maximum-frame-size`:** only raises the ceiling, doesn't decouple size from transport, and changes remoting config globally.
|
||||
- **Sites read central MS SQL directly:** rejected — it tears down the enforced hub-spoke boundary (sites = local SQLite, central = MS SQL, enforced in DI *and* `StartupValidator`), gives every site node a login that can read central's *entire* config DB (huge blast radius), and couples sites to central's EF schema.
|
||||
|
||||
A small `RefreshDeploymentCommand` routes to the active-node singleton **for free** via the existing `ClusterSingletonProxy` (no host detection, no frame risk), and the same small-message pattern fixes replication. The only genuinely new piece is the **fetch channel**, implemented as an authenticated `GET` against central's **existing** HTTP management API.
|
||||
|
||||
## 3. Architecture overview
|
||||
|
||||
```
|
||||
central MS SQL
|
||||
┌──────────────────┐
|
||||
│ PendingDeployment │ (configJson, token, TTL, by deploymentId; ≤1 per instance)
|
||||
└──────────────────┘
|
||||
▲ │ read (token-gated)
|
||||
persist before notify │
|
||||
│ ▼
|
||||
DeploymentService ──Akka small notify──▶ site (active node, via singleton proxy)
|
||||
RefreshDeploymentCommand(id, name, hash, │
|
||||
deployedBy, ts, centralFetchBaseUrl, token) │ 1. HTTP GET config (token) ◀── central mgmt API
|
||||
▲ │ 2. apply (existing path): Instance Actor + SQLite
|
||||
│ DeploymentStatusResponse│ 3. reply success/failure
|
||||
└─────────────────────────┘
|
||||
│ 4. tell standby (small, id-only)
|
||||
▼
|
||||
SiteReplicationActor (standby)
|
||||
HTTP GET config (token) ──▶ central mgmt API
|
||||
write deployed_configurations SQLite (no Instance Actor)
|
||||
```
|
||||
|
||||
Every Akka hop now carries only small control messages. The bulk config moves only over HTTP, point-to-point, to the node that needs it.
|
||||
|
||||
## 4. Component changes
|
||||
|
||||
### 4.1 Central — pending-config store
|
||||
|
||||
New table **`PendingDeployment`** in central MS SQL (EF entity + migration):
|
||||
|
||||
| column | type | notes |
|
||||
|---|---|---|
|
||||
| `DeploymentId` | `varchar` PK | matches the deploy's GUID (`N` format) |
|
||||
| `InstanceId` | `int`, indexed | for supersession lookup |
|
||||
| `RevisionHash` | `varchar` | staleness/version marker |
|
||||
| `ConfigJson` | `nvarchar(max)` | the flattened config |
|
||||
| `Token` | `varchar` | random per-deployment fetch secret |
|
||||
| `CreatedAtUtc` | `datetimeoffset` | |
|
||||
| `ExpiresAtUtc` | `datetimeoffset`, indexed | TTL (default 5 min, configurable) |
|
||||
|
||||
Kept **separate** from `DeployedConfigSnapshot` so the existing "deployed = success" semantics stay clean. On success the pending row's config is promoted into `DeployedConfigSnapshot` (existing logic); on failure/timeout/TTL it's purged. A bounded periodic purge removes expired rows.
|
||||
|
||||
**Supersession (requested):** when persisting a new pending row for an instance, **first delete any existing pending rows for that same `InstanceId`** (delete-then-insert, one transaction) → at most one pending row per instance, always the latest. Race-free because the existing **per-instance operation lock** already serializes same-instance deploys.
|
||||
|
||||
### 4.2 Central — serve endpoint
|
||||
|
||||
`GET /api/internal/deployments/{deploymentId}/config` on the existing management API:
|
||||
|
||||
- `[AllowAnonymous]` + **explicit token validation** (constant-time compare against the pending row's `Token`), TTL-checked. Machine-to-machine; never user-session-gated.
|
||||
- Returns the row's `ConfigJson` (200) — or **404** when the `deploymentId` is unknown, expired, or **superseded** (no longer the current pending row for its instance). The 404-on-superseded is what makes stale fetches fail closed (see §5).
|
||||
- Both central nodes read the same MS SQL, so the request works through Traefik's active-node LB.
|
||||
- HTTPS strongly recommended in transit (config carries already-encrypted `SecretsBlock`s; the token is single-deployment + short-TTL).
|
||||
- Companion endpoint for the follow-up (§6): `GET /api/internal/sites/{siteId}/deployments` → `[{instanceUniqueName, revisionHash}]`.
|
||||
|
||||
### 4.3 Central — send small notify
|
||||
|
||||
New message:
|
||||
|
||||
```csharp
|
||||
public record RefreshDeploymentCommand(
|
||||
string DeploymentId,
|
||||
string InstanceUniqueName,
|
||||
string RevisionHash,
|
||||
string DeployedBy,
|
||||
DateTimeOffset Timestamp,
|
||||
string CentralFetchBaseUrl, // central's LB/Traefik URL — carried so the site needs no new standing config
|
||||
string FetchToken);
|
||||
```
|
||||
|
||||
`DeploymentService` flow: flatten + hash (unchanged) → persist `PendingDeployment` (config + token + TTL, superseding prior) → send `RefreshDeploymentCommand` over the existing ClusterClient path (replacing the fat `DeployInstanceCommand`) → await `DeploymentStatusResponse` (reply path unchanged) → on success promote pending→snapshot. The `Ask`/`DeploymentTimeout` stay, but now carry a tiny request/reply — no frame risk. `CentralFetchBaseUrl` is a new central option (e.g. `Communication:CentralFetchBaseUrl`).
|
||||
|
||||
### 4.4 Site — deploy (active node)
|
||||
|
||||
- `SiteCommunicationActor` routes `RefreshDeploymentCommand` to the `deployment-manager-proxy` (replacing the `DeployInstanceCommand` forward). The proxy delivers to the singleton on the **active node automatically** — small message, no host detection.
|
||||
- `DeploymentManagerActor` handles `RefreshDeploymentCommand`: fetch the config via an injected `IDeploymentConfigFetcher` (HttpClient wrapper, registered for the site role), then run the **existing `ApplyDeployment`** verbatim (create/replace Instance Actor, write `deployed_configurations`, reply `DeploymentStatusResponse`). Only the config source changes.
|
||||
- Fetch failures reply `Failed` with a `Communication failure:`-prefixed message (transient/comm) so central's DeploymentManager-006 query-site reconciliation fires; apply failures stay generic deployment errors.
|
||||
|
||||
### 4.5 Site — replication via fetch (active → standby), unified
|
||||
|
||||
- `ReplicateConfigDeploy` / `ApplyConfigDeploy` **drop `ConfigJson`** and gain `(DeploymentId, RevisionHash, IsEnabled, CentralFetchBaseUrl, FetchToken)`. After applying, the active node sends the id-only message to the standby.
|
||||
- The standby's `SiteReplicationActor` **fetches** the config via the same `IDeploymentConfigFetcher`, then writes `deployed_configurations` via `SiteStorageService.StoreDeployedConfigAsync` — **no Instance Actor** (it's standby). Best-effort, but the fetch gets a small bounded retry (cheap/async).
|
||||
- **Defensive write guard:** the standby only overwrites a `deployed_configurations` row when the incoming `RevisionHash`/`DeployedAt` is newer, so a delayed older replicate can't clobber a newer config.
|
||||
- A stale replicate (superseded deploymentId) fetches → 404 → standby skips it; the newer deploy's replicate-notify delivers the correct config (§5).
|
||||
- These records are intra-site (same Host binary on both nodes) → no cross-version concern.
|
||||
|
||||
### 4.6 Secondary bug — timeout classification
|
||||
|
||||
`DeploymentService.cs` (~line 312): add `AskTimeoutException` to the timeout branch (it does **not** derive from `System.TimeoutException`), alongside `OperationCanceledException`, so a no-reply deploy is classified as a timeout and triggers the query-site reconciliation.
|
||||
|
||||
### 4.7 Retire the fat message
|
||||
|
||||
`DeployInstanceCommand.FlattenedConfigurationJson` is no longer sent cross-cluster. Keep the record only if convenient as an internal apply DTO; otherwise remove. After this change, **no Akka message carries config.**
|
||||
|
||||
## 5. Supersession & ordering semantics
|
||||
|
||||
- **At central:** delete-then-insert under the per-instance lock ⇒ ≤1 pending row per instance, always the newest deploy.
|
||||
- **Fetch fails closed:** the serve endpoint returns 404 for any `deploymentId` that isn't the current pending row (unknown / expired / superseded / already promoted). So an older in-flight deploy notify or an older replicate-notify can never fetch a lower-version config.
|
||||
- **Standby last-write-wins + guard:** `deployed_configurations` upserts by `instance_unique_name`; the standby additionally refuses to overwrite with an older `RevisionHash`/`DeployedAt`. Net: the standby converges on the latest config, never a stale one.
|
||||
|
||||
## 6. Follow-up (in the plan, separate & lower-priority): standby/startup reconciliation
|
||||
|
||||
Replication is best-effort with no retry and **no startup reconciliation** today — a standby that is *down* during a deploy permanently misses that instance until its next deploy (a pre-existing gap, independent of frame size). To make replication self-healing:
|
||||
|
||||
- On node start / peer rejoin, the node reconciles its `deployed_configurations` against central's **expected set for this site** (via `GET /api/internal/sites/{siteId}/deployments` → `{instanceUniqueName, revisionHash}` pairs), fetching missing/stale configs (reusing the §4.2 config endpoint) and dropping orphans central no longer has.
|
||||
|
||||
Marked as a distinct task, sequenced **after** the core frame-size fix.
|
||||
|
||||
## 7. Configuration & options
|
||||
|
||||
- **Central:** `Communication:CentralFetchBaseUrl` (LB/Traefik URL); `Deployment:PendingDeploymentTtl` (default 5 min); token length; purge interval.
|
||||
- **Site:** `HttpClient` registration for the site role; fetch timeout; replication fetch retry count.
|
||||
- **Akka `maximum-frame-size`:** intentionally **unchanged** — the fix doesn't rely on it.
|
||||
- gRPC message size: not involved (HTTP carries the config).
|
||||
|
||||
## 8. Failover & edge cases
|
||||
|
||||
- Site failover: the notify routes to the new active node automatically via the singleton proxy.
|
||||
- Active node dies mid-fetch: central's `Ask` times out → reconciliation.
|
||||
- Central node failover during a fetch: Traefik routes to the active central node; the pending row is in shared MS SQL → still served.
|
||||
- Standby down during deploy: misses the replicate-notify → covered by the §6 follow-up.
|
||||
- Token TTL must comfortably cover both nodes' fetches within the deploy window (default 5 min is ample).
|
||||
|
||||
## 9. Security
|
||||
|
||||
Token-gated internal endpoint; constant-time compare; short TTL; scoped to exactly one deployment's config; not tied to a user session; HTTPS recommended. Config `SecretsBlock`s are already encrypted at rest.
|
||||
|
||||
## 10. Testing
|
||||
|
||||
- **Unit:** `PendingDeployment` store incl. supersession (delete-then-insert) + TTL/purge; token validation (valid / expired / wrong / superseded, constant-time); `IDeploymentConfigFetcher` (200 / 401 / 404 / timeout); `DeploymentService` notify→promote flow; AskTimeout classification; standby fetch-on-replicate + older-write guard.
|
||||
- **Integration:** the exact failing case — a **>128 KB** config deploys end-to-end; standby obtains it via fetch; kill the active node → failover recovers it from the standby's SQLite; redeploy a *newer* version → pending superseded, stale fetch 404s.
|
||||
- **Live smoke (docker):** redeploy the 3rd-composition config that originally hung; confirm both nodes hold it.
|
||||
|
||||
## 11. Migration / deploy
|
||||
|
||||
- Single Host binary serves both roles → one rebuild + redeploy covers central and site. Message-contract changes (`RefreshDeploymentCommand` new; `ReplicateConfigDeploy` drops `ConfigJson`) are safe since all nodes upgrade together.
|
||||
- New EF migration for `PendingDeployment` (auto-apply in dev; SQL script for prod). No site SQLite schema change.
|
||||
- Add `CentralFetchBaseUrl` to central appsettings in `docker/`, `docker-env2/`, `deploy/wonder-app-vd03/`; confirm site→central HTTP reachability (fine on the co-located test host/docker; a firewall port to open in a hub-spoke prod). Update `deploy/wonder-app-vd03/RUNBOOK.md`.
|
||||
|
||||
## 11a. Live validation + post-implementation fixes (2026-06-26)
|
||||
|
||||
Smoke-tested on the docker cluster (rebuilt from this branch). Validated end-to-end:
|
||||
- Migration applied; all 8 nodes healthy/ready.
|
||||
- **Deploy notify-and-fetch**: central notify → active node fetch+apply → id-only replicate → standby fetch → `Success` in **~0.11 s** (the exact path that previously hung 120 s).
|
||||
- **Startup reconciliation**: every node self-heals on startup — single-node gap heals; the **concurrent both-nodes-empty** race heals both.
|
||||
|
||||
The smoke surfaced two real bugs in the reconciliation path (missed by unit/integration tests because those didn't have a second concurrent node or a lingering expired row), both fixed:
|
||||
1. **Concurrent-gap omit** — when two nodes were concurrently missing the same instance, the second node's `StagePendingIfAbsentAsync` returned false and the handler *omitted* the item, leaving that node unhealed. Fix: on false, return the **existing** pending row's deploymentId + token (multi-use within TTL) so all concurrently-missing nodes heal in the same round.
|
||||
2. **Expired pending row blocks self-heal** — `StagePendingIfAbsentAsync` checked existence by `InstanceId` ignoring expiry, so an expired-but-unpurged row (the periodic purge is still a deferred TODO) blocked a fresh stage *and* would collide with the snapshot's reused `DeploymentId` on the unique index. Fix: **expiry-aware staging** — delete expired rows for the instance first, then check only live rows; `GetPendingDeploymentByInstanceIdAsync` filters by expiry. This also opportunistically cleans expired rows, reducing reliance on the deferred periodic purge.
|
||||
|
||||
## 12. Affected files (for the plan)
|
||||
|
||||
- `src/ZB.MOM.WW.ScadaBridge.Commons/Entities/Deployment/` — new `PendingDeployment` entity.
|
||||
- `src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/IDeploymentManagerRepository.cs` — pending CRUD + supersession + expected-set query.
|
||||
- `src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Deployment/` — new `RefreshDeploymentCommand`; retire `DeployInstanceCommand` wire use.
|
||||
- `src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/` — EF mapping, repository impl, migration.
|
||||
- `src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentService.cs` — persist pending + supersede, send notify, promote on success, AskTimeout classification.
|
||||
- `src/ZB.MOM.WW.ScadaBridge.Communication/CommunicationService.cs` + `Actors/{Central,Site}CommunicationActor.cs` — route `RefreshDeploymentCommand`.
|
||||
- `src/ZB.MOM.WW.ScadaBridge.CentralUI` (or management API project) — `/api/internal/deployments/{id}/config` + `/api/internal/sites/{id}/deployments` endpoints.
|
||||
- `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs` — handle `RefreshDeploymentCommand`, fetch, replicate id-only.
|
||||
- `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/SiteReplicationActor.cs` + `Messages/ReplicationMessages.cs` — id-only replication, fetch on standby, older-write guard.
|
||||
- `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/` — `IDeploymentConfigFetcher` + HttpClient registration; site reconciliation (follow-up).
|
||||
- appsettings (`docker/`, `docker-env2/`, `deploy/wonder-app-vd03/`) + `RUNBOOK.md`.
|
||||
- Tests under `tests/`.
|
||||
@@ -0,0 +1,800 @@
|
||||
# Deploy + Replication Config Transport (Notify-and-Fetch) Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Stop shipping the full flattened instance config over Akka on the two hops that carry it today (central→site deploy and site active→standby replication), so a large config can never exceed Akka's 128 KB `maximum-frame-size` and silently break a deploy or a standby.
|
||||
|
||||
**Architecture:** Central persists the flattened config to a new `PendingDeployment` table (MS SQL, ≤1 row per instance, TTL + per-deployment token) and sends a **small** `RefreshDeploymentCommand` over the existing ClusterClient path. It routes — via the existing `ClusterSingletonProxy` — to the `DeploymentManager` singleton on the **active** site node, which **fetches** the config over HTTP from central's existing management API (token-gated `GET /api/internal/deployments/{id}/config`) and runs the unchanged apply path. Replication is unified: the active node sends the standby an **id-only** `ReplicateConfigDeploy`, and the standby fetches the same config over HTTP and persists it (no Instance Actors). No Akka message carries config after this; `maximum-frame-size` is left untouched.
|
||||
|
||||
**Tech Stack:** C#/.NET 10, Akka.NET (ClusterClient, ClusterSingleton), EF Core + MS SQL (central), SQLite (site), ASP.NET Core minimal APIs, xUnit.
|
||||
|
||||
**Design doc:** [`docs/plans/2026-06-26-deploy-config-notify-and-fetch-design.md`](2026-06-26-deploy-config-notify-and-fetch-design.md)
|
||||
**Fixes:** [`docs/known-issues/2026-06-26-deploy-config-exceeds-akka-frame-size.md`](../known-issues/2026-06-26-deploy-config-exceeds-akka-frame-size.md)
|
||||
|
||||
## Conventions
|
||||
|
||||
- **Run tests filtered** (per project memory): `dotnet test <project.csproj> --filter <Name>`; build only affected projects during the loop; full `dotnet build ZB.MOM.WW.ScadaBridge.slnx` once at the end.
|
||||
- **EF migrations** (per project memory): `dotnet build` the ConfigurationDatabase + Host FIRST, then `dotnet ef migrations add <Name> --project src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase --startup-project src/ZB.MOM.WW.ScadaBridge.Host` (NEVER `--no-build`; if the migration scaffolds empty, delete the files and rebuild first).
|
||||
- **All timestamps UTC** (`DateTimeOffset.UtcNow`).
|
||||
- **Commit after each task** with a descriptive message.
|
||||
|
||||
## Task dependency overview
|
||||
|
||||
```
|
||||
Foundation (parallel): T1 (entity+migration) T3 (RefreshDeploymentCommand) T4 (options+token)
|
||||
Central store: T2 (repo) ── needs T1
|
||||
Central serve: T5 (endpoint) ── needs T1,T2,T4
|
||||
Central send: T6 (DeploymentService) ── needs T2,T3,T4 then T8 (AskTimeout, same file)
|
||||
T7 (CommunicationService route) ── needs T3
|
||||
Site fetch+apply: T9 (fetcher+DI) ── needs T4 then T10 (DM actor) ── needs T3,T9 T11 (SiteComm route) ── needs T3
|
||||
Replication: T13 (storage guard) then T12 (replication msgs+actors) ── needs T9,T13
|
||||
Cleanup/config: T14 (retire wire use) ── needs T6,T7,T10,T11 T15 (appsettings/RUNBOOK)
|
||||
Verify: T16 (integration test) ── needs core T17 (live smoke) ── needs T16
|
||||
Follow-up (LAST): T18 (standby/startup reconciliation) ── needs all core
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `PendingDeployment` entity + EF mapping + migration
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 3, Task 4
|
||||
|
||||
**Files:**
|
||||
- Create: `src/ZB.MOM.WW.ScadaBridge.Commons/Entities/Deployment/PendingDeployment.cs`
|
||||
- Create: `src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Configurations/PendingDeploymentConfiguration.cs`
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ScadaBridgeDbContext.cs` (add `DbSet`, after the `DeployedConfigSnapshots` set at ~line 91)
|
||||
- Create (generated): `src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Migrations/<timestamp>_AddPendingDeployment.cs`
|
||||
|
||||
**Step 1: Write the entity** (mirror `DeployedConfigSnapshot.cs` style):
|
||||
|
||||
```csharp
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// A flattened instance config staged for a single in-flight deployment, fetched
|
||||
/// by the site over HTTP instead of being shipped inside an Akka message (avoids
|
||||
/// the 128 KB frame limit). Keyed by DeploymentId; at most one row per InstanceId
|
||||
/// (a newer deploy supersedes the prior pending row). Carries a per-deployment
|
||||
/// fetch token and a TTL. Promoted to DeployedConfigSnapshot on success; purged
|
||||
/// on failure/timeout/TTL.
|
||||
/// </summary>
|
||||
public class PendingDeployment
|
||||
{
|
||||
public int Id { get; set; }
|
||||
public string DeploymentId { get; set; }
|
||||
public int InstanceId { get; set; }
|
||||
public string RevisionHash { get; set; }
|
||||
public string ConfigurationJson { get; set; }
|
||||
public string Token { get; set; }
|
||||
public DateTimeOffset CreatedAtUtc { get; set; }
|
||||
public DateTimeOffset ExpiresAtUtc { get; set; }
|
||||
|
||||
public PendingDeployment(
|
||||
string deploymentId, int instanceId, string revisionHash,
|
||||
string configurationJson, string token,
|
||||
DateTimeOffset createdAtUtc, DateTimeOffset expiresAtUtc)
|
||||
{
|
||||
DeploymentId = deploymentId ?? throw new ArgumentNullException(nameof(deploymentId));
|
||||
InstanceId = instanceId;
|
||||
RevisionHash = revisionHash ?? throw new ArgumentNullException(nameof(revisionHash));
|
||||
ConfigurationJson = configurationJson ?? throw new ArgumentNullException(nameof(configurationJson));
|
||||
Token = token ?? throw new ArgumentNullException(nameof(token));
|
||||
CreatedAtUtc = createdAtUtc;
|
||||
ExpiresAtUtc = expiresAtUtc;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Write the EF configuration** (look at an existing file in `Configurations/` for column-type conventions — `nvarchar(max)` for JSON, unique index on `DeploymentId`, index on `InstanceId` and `ExpiresAtUtc`):
|
||||
|
||||
```csharp
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Configurations;
|
||||
|
||||
public class PendingDeploymentConfiguration : IEntityTypeConfiguration<PendingDeployment>
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<PendingDeployment> b)
|
||||
{
|
||||
b.ToTable("PendingDeployments");
|
||||
b.HasKey(x => x.Id);
|
||||
b.Property(x => x.DeploymentId).IsRequired().HasMaxLength(64);
|
||||
b.HasIndex(x => x.DeploymentId).IsUnique();
|
||||
b.HasIndex(x => x.InstanceId);
|
||||
b.HasIndex(x => x.ExpiresAtUtc);
|
||||
b.Property(x => x.RevisionHash).IsRequired().HasMaxLength(128);
|
||||
b.Property(x => x.ConfigurationJson).IsRequired(); // nvarchar(max)
|
||||
b.Property(x => x.Token).IsRequired().HasMaxLength(128);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Step 3: Add the DbSet** in `ScadaBridgeDbContext.cs` (after `DeployedConfigSnapshots`):
|
||||
|
||||
```csharp
|
||||
public DbSet<PendingDeployment> PendingDeployments => Set<PendingDeployment>();
|
||||
```
|
||||
|
||||
**Step 4: Build, then add the migration** (memory: build first, never `--no-build`):
|
||||
|
||||
```bash
|
||||
dotnet build src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.csproj
|
||||
dotnet build src/ZB.MOM.WW.ScadaBridge.Host/ZB.MOM.WW.ScadaBridge.Host.csproj
|
||||
dotnet ef migrations add AddPendingDeployment \
|
||||
--project src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase \
|
||||
--startup-project src/ZB.MOM.WW.ScadaBridge.Host
|
||||
```
|
||||
Expected: a non-empty migration creating `PendingDeployments`. If it scaffolds empty, delete the generated files, rebuild, retry.
|
||||
|
||||
**Step 5: Commit** — `feat(deploy): add PendingDeployment entity + migration`
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Pending-deployment repository methods (with supersession)
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (depends on Task 1)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/IDeploymentManagerRepository.cs:109-137` (add alongside the snapshot methods)
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/DeploymentManagerRepository.cs` (implement)
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/PendingDeploymentRepositoryTests.cs` (use the existing in-memory/SQLite test DbContext fixture pattern in that test project)
|
||||
|
||||
**Step 1: Write failing tests** — supersession + lookup + purge:
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task AddPending_supersedes_prior_row_for_same_instance()
|
||||
{
|
||||
await using var ctx = NewContext();
|
||||
var repo = new DeploymentManagerRepository(ctx);
|
||||
await repo.AddPendingDeploymentAsync(new PendingDeployment("dep1", 7, "h1", "{old}", "t1", Now, Now.AddMinutes(5)));
|
||||
await repo.SaveChangesAsync();
|
||||
await repo.AddPendingDeploymentAsync(new PendingDeployment("dep2", 7, "h2", "{new}", "t2", Now, Now.AddMinutes(5)));
|
||||
await repo.SaveChangesAsync();
|
||||
|
||||
Assert.Null(await repo.GetPendingDeploymentByIdAsync("dep1")); // superseded
|
||||
var current = await repo.GetPendingDeploymentByIdAsync("dep2");
|
||||
Assert.Equal("{new}", current!.ConfigurationJson);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PurgeExpired_removes_only_expired()
|
||||
{
|
||||
await using var ctx = NewContext();
|
||||
var repo = new DeploymentManagerRepository(ctx);
|
||||
await repo.AddPendingDeploymentAsync(new PendingDeployment("live", 1, "h", "{}", "t", Now, Now.AddMinutes(5)));
|
||||
await repo.AddPendingDeploymentAsync(new PendingDeployment("dead", 2, "h", "{}", "t", Now.AddMinutes(-10), Now.AddMinutes(-5)));
|
||||
await repo.SaveChangesAsync();
|
||||
var removed = await repo.PurgeExpiredPendingDeploymentsAsync(Now);
|
||||
Assert.Equal(1, removed);
|
||||
Assert.NotNull(await repo.GetPendingDeploymentByIdAsync("live"));
|
||||
Assert.Null(await repo.GetPendingDeploymentByIdAsync("dead"));
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run, verify they fail** (methods don't exist): `dotnet test tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/...csproj --filter PendingDeployment`
|
||||
|
||||
**Step 3: Add interface methods:**
|
||||
|
||||
```csharp
|
||||
// Notify-and-fetch: PendingDeployment staging store.
|
||||
Task AddPendingDeploymentAsync(PendingDeployment pending, CancellationToken cancellationToken = default);
|
||||
Task<PendingDeployment?> GetPendingDeploymentByIdAsync(string deploymentId, CancellationToken cancellationToken = default);
|
||||
Task DeletePendingDeploymentByIdAsync(string deploymentId, CancellationToken cancellationToken = default);
|
||||
Task<int> PurgeExpiredPendingDeploymentsAsync(DateTimeOffset nowUtc, CancellationToken cancellationToken = default);
|
||||
// Follow-up (Task 18): expected deployed set per site for reconciliation.
|
||||
Task<IReadOnlyList<(string InstanceUniqueName, string RevisionHash)>> GetExpectedDeploymentsForSiteAsync(int siteId, CancellationToken cancellationToken = default);
|
||||
```
|
||||
|
||||
**Step 4: Implement** — `AddPendingDeploymentAsync` does supersession (delete prior rows for the same `InstanceId` before adding); safe because the per-instance operation lock serializes same-instance deploys:
|
||||
|
||||
```csharp
|
||||
public async Task AddPendingDeploymentAsync(PendingDeployment pending, CancellationToken ct = default)
|
||||
{
|
||||
var prior = await _dbContext.Set<PendingDeployment>()
|
||||
.Where(p => p.InstanceId == pending.InstanceId).ToListAsync(ct);
|
||||
if (prior.Count > 0) _dbContext.Set<PendingDeployment>().RemoveRange(prior);
|
||||
await _dbContext.Set<PendingDeployment>().AddAsync(pending, ct);
|
||||
}
|
||||
|
||||
public Task<PendingDeployment?> GetPendingDeploymentByIdAsync(string deploymentId, CancellationToken ct = default)
|
||||
=> _dbContext.Set<PendingDeployment>().FirstOrDefaultAsync(p => p.DeploymentId == deploymentId, ct);
|
||||
|
||||
public async Task DeletePendingDeploymentByIdAsync(string deploymentId, CancellationToken ct = default)
|
||||
{
|
||||
var row = await _dbContext.Set<PendingDeployment>().FirstOrDefaultAsync(p => p.DeploymentId == deploymentId, ct);
|
||||
if (row != null) _dbContext.Set<PendingDeployment>().Remove(row);
|
||||
}
|
||||
|
||||
public async Task<int> PurgeExpiredPendingDeploymentsAsync(DateTimeOffset nowUtc, CancellationToken ct = default)
|
||||
{
|
||||
var expired = await _dbContext.Set<PendingDeployment>().Where(p => p.ExpiresAtUtc <= nowUtc).ToListAsync(ct);
|
||||
if (expired.Count == 0) return 0;
|
||||
_dbContext.Set<PendingDeployment>().RemoveRange(expired);
|
||||
await _dbContext.SaveChangesAsync(ct);
|
||||
return expired.Count;
|
||||
}
|
||||
```
|
||||
`GetExpectedDeploymentsForSiteAsync`: join `Instances` (by `SiteId`) with the latest `DeploymentRecord`/`DeployedConfigSnapshot` to return `(UniqueName, RevisionHash)` for instances that have a successful deployed snapshot. (Used only by Task 18 — a minimal correct query is fine here.)
|
||||
|
||||
**Step 5: Run tests — PASS. Commit** — `feat(deploy): pending-deployment repository with supersession + purge`
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `RefreshDeploymentCommand` message contract
|
||||
|
||||
**Classification:** trivial
|
||||
**Estimated implement time:** ~2 min
|
||||
**Parallelizable with:** Task 1, Task 4
|
||||
|
||||
**Files:**
|
||||
- Create: `src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Deployment/RefreshDeploymentCommand.cs`
|
||||
|
||||
**Step 1: Write the record** (mirror `DeployInstanceCommand.cs`; note: **no** config JSON):
|
||||
|
||||
```csharp
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// Small central→site notify (replaces the fat DeployInstanceCommand on the wire).
|
||||
/// Routes via the deployment-manager singleton proxy to the active node, which
|
||||
/// fetches the flattened config from <paramref name="CentralFetchBaseUrl"/> using
|
||||
/// <paramref name="FetchToken"/> and applies it. Reply is the existing
|
||||
/// DeploymentStatusResponse.
|
||||
/// </summary>
|
||||
public record RefreshDeploymentCommand(
|
||||
string DeploymentId,
|
||||
string InstanceUniqueName,
|
||||
string RevisionHash,
|
||||
string DeployedBy,
|
||||
DateTimeOffset Timestamp,
|
||||
string CentralFetchBaseUrl,
|
||||
string FetchToken);
|
||||
```
|
||||
|
||||
**Step 2: Build the Commons project.** Expected: compiles.
|
||||
|
||||
**Step 3: Commit** — `feat(deploy): add RefreshDeploymentCommand notify message`
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Fetch options + token generation
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Task 1, Task 3
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.Communication/CommunicationOptions.cs` (add `CentralFetchBaseUrl`, `PendingDeploymentTtl`)
|
||||
- Create: `src/ZB.MOM.WW.ScadaBridge.Commons/Types/Deployment/DeploymentFetchToken.cs` (token gen + constant-time compare helper)
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/DeploymentFetchTokenTests.cs`
|
||||
|
||||
**Step 1: Failing test** for the constant-time compare + token shape:
|
||||
|
||||
```csharp
|
||||
[Fact] public void Generated_token_is_url_safe_and_long() {
|
||||
var t = DeploymentFetchToken.Generate();
|
||||
Assert.True(t.Length >= 32);
|
||||
Assert.DoesNotContain('+', t); Assert.DoesNotContain('/', t);
|
||||
}
|
||||
[Fact] public void Equals_is_true_only_for_match() {
|
||||
var t = DeploymentFetchToken.Generate();
|
||||
Assert.True(DeploymentFetchToken.ConstantTimeEquals(t, t));
|
||||
Assert.False(DeploymentFetchToken.ConstantTimeEquals(t, t + "x"));
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Implement helper** (`RandomNumberGenerator` + `CryptographicOperations.FixedTimeEquals`):
|
||||
|
||||
```csharp
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
public static class DeploymentFetchToken
|
||||
{
|
||||
public static string Generate() =>
|
||||
Convert.ToBase64String(RandomNumberGenerator.GetBytes(32))
|
||||
.Replace('+', '-').Replace('/', '_').TrimEnd('=');
|
||||
public static bool ConstantTimeEquals(string a, string b) =>
|
||||
CryptographicOperations.FixedTimeEquals(
|
||||
Encoding.UTF8.GetBytes(a), Encoding.UTF8.GetBytes(b));
|
||||
}
|
||||
```
|
||||
|
||||
**Step 3: Add options** to `CommunicationOptions.cs`:
|
||||
|
||||
```csharp
|
||||
/// Base URL (Traefik/LB) the SITE uses to fetch deploy configs from central. e.g. "https://central.example:9000".
|
||||
public string CentralFetchBaseUrl { get; set; } = "";
|
||||
/// How long a staged PendingDeployment (and its fetch token) stays valid. Must cover both nodes' fetches.
|
||||
public TimeSpan PendingDeploymentTtl { get; set; } = TimeSpan.FromMinutes(5);
|
||||
```
|
||||
|
||||
**Step 4: Run token tests — PASS. Commit** — `feat(deploy): fetch options + per-deployment token helper`
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Central serve endpoint `GET /api/internal/deployments/{deploymentId}/config`
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (depends on Task 1, 2, 4)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/ZB.MOM.WW.ScadaBridge.ManagementService/DeploymentConfigEndpoints.cs` (mirror `AuditEndpoints.cs` minimal-API style)
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.Host/Program.cs:367` area (central role block — add `app.MapDeploymentConfigAPI();` next to `MapAuditAPI()`)
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/DeploymentConfigEndpointsTests.cs` (or a `WebApplicationFactory`-based test if that pattern exists; otherwise unit-test the handler delegate directly with a fake repository)
|
||||
|
||||
**Step 1: Failing test** — handler returns 200+config for a valid token, 404 for unknown/expired/superseded, 404 (not 401, to avoid id enumeration) or 401 for a wrong token. Decide: return **404** for unknown id and **401** for known-id-but-bad-token; both for expired → 404. Test those three.
|
||||
|
||||
**Step 2: Implement the endpoint** (`[AllowAnonymous]` equivalent — minimal-API endpoints are anonymous by default unless a global auth filter applies; if a global requirement exists, add `.AllowAnonymous()`):
|
||||
|
||||
```csharp
|
||||
public static class DeploymentConfigEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapDeploymentConfigAPI(this IEndpointRouteBuilder endpoints)
|
||||
{
|
||||
endpoints.MapGet("/api/internal/deployments/{deploymentId}/config", HandleGetConfig)
|
||||
.AllowAnonymous();
|
||||
return endpoints;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleGetConfig(
|
||||
string deploymentId, HttpContext ctx,
|
||||
IDeploymentManagerRepository repo, IClock clock, CancellationToken ct) // use existing time abstraction or DateTimeOffset.UtcNow
|
||||
{
|
||||
var token = ctx.Request.Headers["X-Deployment-Token"].ToString();
|
||||
var row = await repo.GetPendingDeploymentByIdAsync(deploymentId, ct);
|
||||
if (row is null) return Results.NotFound(); // unknown or superseded
|
||||
if (row.ExpiresAtUtc <= DateTimeOffset.UtcNow) return Results.NotFound(); // expired
|
||||
if (string.IsNullOrEmpty(token) || !DeploymentFetchToken.ConstantTimeEquals(token, row.Token))
|
||||
return Results.Unauthorized();
|
||||
return Results.Content(row.ConfigurationJson, "application/json");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Step 3: Map it** in `Program.cs` central block (next to `app.MapAuditAPI();`).
|
||||
|
||||
**Step 4: Run tests — PASS.** Manually note: this endpoint must be reachable via the same host/port as `/management` and `/api/audit/*`.
|
||||
|
||||
**Step 5: Commit** — `feat(deploy): token-gated internal deployment-config fetch endpoint`
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Central send path — stage pending + send notify + promote
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (depends on Task 2, 3, 4)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentService.cs:196-298` (replace the build-command/send/post-success region)
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests/DeploymentServiceTests.cs` (extend existing; the project already fakes `CommunicationService`/repository)
|
||||
|
||||
**Step 1: Failing test** — on deploy, a `PendingDeployment` is staged (with config + token) BEFORE the notify is sent; a `RefreshDeploymentCommand` (not `DeployInstanceCommand`) is sent; on Success the pending row is deleted/promoted to a snapshot; on failure the pending row is purged.
|
||||
|
||||
**Step 2: Implement.** After computing `configJson` (line 198) and resolving `siteId` (line 240):
|
||||
|
||||
```csharp
|
||||
var token = DeploymentFetchToken.Generate();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
await _repository.AddPendingDeploymentAsync(new PendingDeployment(
|
||||
deploymentId, instanceId, revisionHash, configJson, token,
|
||||
now, now + _options.PendingDeploymentTtl), cancellationToken);
|
||||
await _repository.SaveChangesAsync(cancellationToken);
|
||||
|
||||
var refresh = new RefreshDeploymentCommand(
|
||||
deploymentId, instance.UniqueName, revisionHash, user, now,
|
||||
_options.CentralFetchBaseUrl, token);
|
||||
|
||||
var response = await _communicationService.RefreshDeploymentAsync(siteId, refresh, cancellationToken);
|
||||
```
|
||||
- Replace `_communicationService.DeployInstanceAsync(...)` with `RefreshDeploymentAsync` (Task 7). `_options` here is the DeploymentManager's options; if `CentralFetchBaseUrl`/`PendingDeploymentTtl` live on `CommunicationOptions`, inject those (or duplicate onto the deployment options — keep them on `CommunicationOptions` and inject `IOptions<CommunicationOptions>` into `DeploymentService`).
|
||||
- On `DeploymentStatus.Success` (existing `ApplyPostSuccessSideEffectsAsync` path): also `await _repository.DeletePendingDeploymentByIdAsync(deploymentId, ...)` AFTER the snapshot is written (the standby may still need to fetch — see note). **Note on standby timing:** the standby fetches via the replicate-notify which the active node sends right after its own apply; that happens well within the TTL. To be safe, **do not delete the pending row on success** — let the TTL purge it (a background purge, Task added in §Purge below) — so a slightly-delayed standby fetch still succeeds. Promote to snapshot, leave pending to expire. Adjust the test accordingly (assert snapshot written; pending row left until TTL).
|
||||
- In the `catch` block, purge the pending row with `CancellationToken.None` (best-effort) so a failed deploy doesn't leave a fetchable stale config.
|
||||
|
||||
**Step 3: Add a bounded periodic purge.** Wire `PurgeExpiredPendingDeploymentsAsync` into the existing central maintenance/purge cadence (find where `DeployedConfigSnapshot`/audit purge runs; if none convenient, add a simple timer in the central host). Keep it small; if no clean hook exists, leave a TODO and rely on supersession + short TTL (note it in the commit).
|
||||
|
||||
**Step 4: Run tests — PASS. Commit** — `feat(deploy): stage pending config + send RefreshDeploymentCommand`
|
||||
|
||||
---
|
||||
|
||||
### Task 7: `CommunicationService.RefreshDeploymentAsync` + routing
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Task 5, Task 6 (different files: Communication project)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.Communication/CommunicationService.cs:116-135` (add method beside `DeployInstanceAsync`)
|
||||
- Verify: `src/ZB.MOM.WW.ScadaBridge.Communication/Actors/CentralCommunicationActor.cs` (the `SiteEnvelope` routing already forwards any wrapped message to `/user/site-communication` — confirm `RefreshDeploymentCommand` needs no special case)
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/` (extend if a routing test exists)
|
||||
|
||||
**Step 1: Add the method:**
|
||||
|
||||
```csharp
|
||||
public async Task<DeploymentStatusResponse> RefreshDeploymentAsync(
|
||||
string siteId, RefreshDeploymentCommand command, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Sending RefreshDeploymentCommand to site {SiteId}, instance={Instance}, deploymentId={DeploymentId}",
|
||||
siteId, command.InstanceUniqueName, command.DeploymentId);
|
||||
var envelope = new SiteEnvelope(siteId, command);
|
||||
return await GetActor().Ask<DeploymentStatusResponse>(
|
||||
envelope, _options.DeploymentTimeout, cancellationToken);
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Confirm `CentralCommunicationActor`** forwards `SiteEnvelope(_, RefreshDeploymentCommand)` over ClusterClient to `/user/site-communication` (it routes by envelope, message-type-agnostic — verified at `CentralCommunicationActor.cs:361-363`). No change expected; if it has a per-type switch, add `RefreshDeploymentCommand`.
|
||||
|
||||
**Step 3: Build the Communication project. Commit** — `feat(deploy): RefreshDeploymentAsync send method`
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Fix `AskTimeoutException` classification
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** none (same file as Task 6 — sequence AFTER Task 6)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentService.cs:312`
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests/DeploymentServiceTests.cs`
|
||||
|
||||
**Step 1: Failing test** — when the site Ask throws `Akka.Actor.AskTimeoutException`, the record's `ErrorMessage` starts with the timeout prefix (so DeploymentManager-006 reconciliation keying works), not `"Deployment error:"`.
|
||||
|
||||
**Step 2: Implement** — add `AskTimeoutException` to the classification (`using Akka.Actor;`):
|
||||
|
||||
```csharp
|
||||
var isTimeout = ex is TimeoutException or OperationCanceledException or Akka.Actor.AskTimeoutException;
|
||||
```
|
||||
|
||||
**Step 3: Run tests — PASS. Commit** — `fix(deploy): classify AskTimeoutException as a deploy timeout`
|
||||
|
||||
---
|
||||
|
||||
### Task 9: `IDeploymentConfigFetcher` (HTTP) + site DI + options
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 5, Task 7 (different project)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Deployment/IDeploymentConfigFetcher.cs`
|
||||
- Create: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Deployment/HttpDeploymentConfigFetcher.cs`
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ServiceCollectionExtensions.cs:29-67` (`AddSiteRuntime` — add `AddHttpClient` + register fetcher)
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs` (add `ConfigFetchTimeoutSeconds`, `ConfigFetchRetryCount`)
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/HttpDeploymentConfigFetcherTests.cs` (fake `HttpMessageHandler`)
|
||||
|
||||
**Step 1: Failing tests** — fetcher returns the body on 200; throws a typed `DeploymentConfigFetchException` on 401/404/timeout; sends the `X-Deployment-Token` header; builds the URL `"{baseUrl}/api/internal/deployments/{id}/config"`.
|
||||
|
||||
**Step 2: Interface + impl:**
|
||||
|
||||
```csharp
|
||||
public interface IDeploymentConfigFetcher
|
||||
{
|
||||
/// Fetches the flattened config JSON for a deployment from central. Throws
|
||||
/// DeploymentConfigFetchException on any non-success (caller maps to a
|
||||
/// Communication-failure deploy result). 'superseded'/'expired' surface as 404.
|
||||
Task<string> FetchAsync(string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct);
|
||||
}
|
||||
|
||||
public sealed class DeploymentConfigFetchException : Exception
|
||||
{
|
||||
public bool IsSuperseded { get; } // true on 404
|
||||
public DeploymentConfigFetchException(string msg, bool superseded, Exception? inner = null)
|
||||
: base(msg, inner) => IsSuperseded = superseded;
|
||||
}
|
||||
|
||||
public sealed class HttpDeploymentConfigFetcher(HttpClient http, ILogger<HttpDeploymentConfigFetcher> log)
|
||||
: IDeploymentConfigFetcher
|
||||
{
|
||||
public async Task<string> FetchAsync(string baseUrl, string deploymentId, string token, CancellationToken ct)
|
||||
{
|
||||
var url = $"{baseUrl.TrimEnd('/')}/api/internal/deployments/{Uri.EscapeDataString(deploymentId)}/config";
|
||||
using var req = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
req.Headers.Add("X-Deployment-Token", token);
|
||||
HttpResponseMessage resp;
|
||||
try { resp = await http.SendAsync(req, ct); }
|
||||
catch (Exception ex) { throw new DeploymentConfigFetchException($"fetch transport error: {ex.Message}", false, ex); }
|
||||
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
|
||||
throw new DeploymentConfigFetchException("config not found (expired/superseded)", true);
|
||||
if (!resp.IsSuccessStatusCode)
|
||||
throw new DeploymentConfigFetchException($"fetch failed: {(int)resp.StatusCode}", false);
|
||||
return await resp.Content.ReadAsStringAsync(ct);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Step 3: Register** in `AddSiteRuntime` (after the existing registrations ~line 64):
|
||||
|
||||
```csharp
|
||||
services.AddHttpClient<IDeploymentConfigFetcher, HttpDeploymentConfigFetcher>()
|
||||
.ConfigureHttpClient((sp, c) =>
|
||||
c.Timeout = TimeSpan.FromSeconds(
|
||||
sp.GetRequiredService<IOptions<SiteRuntimeOptions>>().Value.ConfigFetchTimeoutSeconds));
|
||||
```
|
||||
Add `ConfigFetchTimeoutSeconds = 30` and `ConfigFetchRetryCount = 3` to `SiteRuntimeOptions`.
|
||||
|
||||
**Step 4: Run tests — PASS. Commit** — `feat(site): HTTP deployment-config fetcher + DI`
|
||||
|
||||
---
|
||||
|
||||
### Task 10: `DeploymentManagerActor` handles `RefreshDeploymentCommand` (fetch → apply)
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 11 (different file)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs:100-132` (ctor param + Receive), and add a `HandleRefreshDeployment` handler near `HandleDeploy:359`
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs:843-855` (pass the fetcher into `Props.Create`)
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/DeploymentManagerActorTests.cs` (TestKit; fake `IDeploymentConfigFetcher`)
|
||||
|
||||
**Step 1: Failing test** — sending `RefreshDeploymentCommand` causes a fetch then the same apply as before (Instance Actor created, SQLite written, `DeploymentStatusResponse(Success)` replied to sender); a fetch failure replies `Failed` with a `Communication failure:`-prefixed message and creates **no** Instance Actor.
|
||||
|
||||
**Step 2: Add the fetcher** as a new optional ctor param (after `loggerFactory`), store `_configFetcher`; add `Receive<RefreshDeploymentCommand>(HandleRefreshDeployment);` beside the existing `Receive<DeployInstanceCommand>`.
|
||||
|
||||
**Step 3: Implement the handler** — fetch off-thread, pipe an internal `RefreshFetched` message back to self carrying the fetched config + original sender, then reuse the existing apply by constructing the internal `DeployInstanceCommand` DTO (kept for this purpose):
|
||||
|
||||
```csharp
|
||||
private sealed record RefreshFetched(RefreshDeploymentCommand Cmd, string ConfigJson, IActorRef Sender);
|
||||
private sealed record RefreshFetchFailed(RefreshDeploymentCommand Cmd, string Error, IActorRef Sender);
|
||||
|
||||
private void HandleRefreshDeployment(RefreshDeploymentCommand cmd)
|
||||
{
|
||||
var sender = Sender;
|
||||
_configFetcher!.FetchAsync(cmd.CentralFetchBaseUrl, cmd.DeploymentId, cmd.FetchToken, CancellationToken.None)
|
||||
.ContinueWith(t => t.IsCompletedSuccessfully
|
||||
? (object)new RefreshFetched(cmd, t.Result, sender)
|
||||
: new RefreshFetchFailed(cmd, t.Exception?.GetBaseException().Message ?? "fetch failed", sender))
|
||||
.PipeTo(Self);
|
||||
}
|
||||
```
|
||||
Register `Receive<RefreshFetched>` → build `new DeployInstanceCommand(cmd.DeploymentId, cmd.InstanceUniqueName, cmd.RevisionHash, ConfigJson, cmd.DeployedBy, cmd.Timestamp)` and route into the **existing** `HandleDeploy` logic (so redeploy buffering etc. is unchanged), preserving `RefreshFetched.Sender` as the reply target. Register `Receive<RefreshFetchFailed>` → reply `new DeploymentStatusResponse(cmd.DeploymentId, cmd.InstanceUniqueName, DeploymentStatus.Failed, $"Communication failure: {Error}", DateTimeOffset.UtcNow)` to `Sender`.
|
||||
- **Refactor note:** `HandleDeploy` currently reads `Sender`. Extract the reply-target so it can be driven by `RefreshFetched.Sender` (the original ClusterClient sender), since by the time `RefreshFetched` arrives `Sender` is `Self`. Thread the sender through the existing redeploy-buffer path.
|
||||
|
||||
**Step 4: Wire Props** in `AkkaHostedService` — resolve `IDeploymentConfigFetcher` from `_serviceProvider` and pass it to `Props.Create(() => new DeploymentManagerActor(... , configFetcher))`.
|
||||
|
||||
**Step 5: Run tests — PASS. Commit** — `feat(site): handle RefreshDeploymentCommand via fetch-then-apply`
|
||||
|
||||
---
|
||||
|
||||
### Task 11: `SiteCommunicationActor` routes `RefreshDeploymentCommand`
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** Task 10 (different file)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteCommunicationActor.cs:95-99` (add a `Receive<RefreshDeploymentCommand>` that forwards to `_deploymentManagerProxy`, mirroring the existing `DeployInstanceCommand` forward)
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/` (TestKit forward test if present)
|
||||
|
||||
**Step 1:** Add:
|
||||
|
||||
```csharp
|
||||
Receive<RefreshDeploymentCommand>(msg =>
|
||||
{
|
||||
_log.Debug("Routing RefreshDeploymentCommand for {0} to DeploymentManager", msg.InstanceUniqueName);
|
||||
_deploymentManagerProxy.Forward(msg);
|
||||
});
|
||||
```
|
||||
Leave the existing `Receive<DeployInstanceCommand>` in place until Task 14 confirms it's dead.
|
||||
|
||||
**Step 2: Build. Commit** — `feat(site): route RefreshDeploymentCommand to deployment-manager proxy`
|
||||
|
||||
---
|
||||
|
||||
### Task 13: Site storage older-write guard
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Task 10, Task 11
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Persistence/SiteStorageService.cs:200-230` (add a guarded variant used by replication)
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/SiteStorageServiceTests.cs`
|
||||
|
||||
**Step 1: Failing test** — `StoreDeployedConfigIfNewerAsync` overwrites when `deployedAt` is newer, and is a no-op when the existing row's `deployed_at` is newer-or-equal.
|
||||
|
||||
**Step 2: Implement** a guarded write (add `WHERE` clause on the upsert, or read-then-conditional-write in the same transaction):
|
||||
|
||||
```sql
|
||||
INSERT INTO deployed_configurations (...) VALUES (...)
|
||||
ON CONFLICT(instance_unique_name) DO UPDATE SET
|
||||
config_json=excluded.config_json, deployment_id=excluded.deployment_id,
|
||||
revision_hash=excluded.revision_hash, is_enabled=excluded.is_enabled,
|
||||
deployed_at=excluded.deployed_at
|
||||
WHERE excluded.deployed_at > deployed_configurations.deployed_at;
|
||||
```
|
||||
Keep the existing unguarded `StoreDeployedConfigAsync` for the active-node apply path (it must always win); use the guarded one only in `SiteReplicationActor` (Task 12).
|
||||
|
||||
**Step 3: Run tests — PASS. Commit** — `feat(site): older-write guard for replicated config writes`
|
||||
|
||||
---
|
||||
|
||||
### Task 12: Unify replication on notify-and-fetch (id-only + standby fetch)
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (depends on Task 9, Task 13)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Messages/ReplicationMessages.cs:10-11,29-30` (drop `ConfigJson`, add fetch coords)
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs:467-470` (send id-only `ReplicateConfigDeploy` with fetch coords; the actor already has the `RefreshDeploymentCommand` fields in scope on the refresh path)
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/SiteReplicationActor.cs:37-50,137-147` (ctor: add `IDeploymentConfigFetcher`; `HandleApplyConfigDeploy`: fetch then guarded write)
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs` (pass fetcher into the `SiteReplicationActor` Props)
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/SiteReplicationActorTests.cs` (fake fetcher)
|
||||
|
||||
**Step 1: Failing test** — given an `ApplyConfigDeploy(deploymentId, …, baseUrl, token)`, the standby fetches via `IDeploymentConfigFetcher` and writes via the guarded store; a 404 (superseded) fetch results in **no** write and is logged, not thrown.
|
||||
|
||||
**Step 2: Change the records** (no `ConfigJson`):
|
||||
|
||||
```csharp
|
||||
public record ReplicateConfigDeploy(
|
||||
string InstanceName, string DeploymentId, string RevisionHash, bool IsEnabled,
|
||||
string CentralFetchBaseUrl, string FetchToken);
|
||||
|
||||
public record ApplyConfigDeploy(
|
||||
string InstanceName, string DeploymentId, string RevisionHash, bool IsEnabled,
|
||||
string CentralFetchBaseUrl, string FetchToken);
|
||||
```
|
||||
|
||||
**Step 3: Update the active-node send** — on the refresh/apply success path in `DeploymentManagerActor`, send `new ReplicateConfigDeploy(instanceName, cmd.DeploymentId, cmd.RevisionHash, true, cmd.CentralFetchBaseUrl, cmd.FetchToken)` (the `SiteReplicationActor` maps it to `ApplyConfigDeploy` as today at `SiteReplicationActor.cs:57-58`).
|
||||
|
||||
**Step 4: Update the standby handler** `HandleApplyConfigDeploy` to fetch then guarded-write:
|
||||
|
||||
```csharp
|
||||
private void HandleApplyConfigDeploy(ApplyConfigDeploy msg)
|
||||
{
|
||||
_configFetcher.FetchAsync(msg.CentralFetchBaseUrl, msg.DeploymentId, msg.FetchToken, CancellationToken.None)
|
||||
.ContinueWith(async t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
{
|
||||
var ex = t.Exception!.GetBaseException();
|
||||
if (ex is DeploymentConfigFetchException { IsSuperseded: true })
|
||||
_logger.LogInformation("Skip replicated {Instance}: config superseded/expired", msg.InstanceName);
|
||||
else
|
||||
_logger.LogError(ex, "Replicated fetch failed for {Instance}", msg.InstanceName);
|
||||
return;
|
||||
}
|
||||
await _storage.StoreDeployedConfigIfNewerAsync(
|
||||
msg.InstanceName, t.Result, msg.DeploymentId, msg.RevisionHash, msg.IsEnabled);
|
||||
}).Unwrap();
|
||||
}
|
||||
```
|
||||
(Best-effort retry up to `ConfigFetchRetryCount` is optional; the Task 18 reconciliation is the durable backstop.)
|
||||
|
||||
**Step 5: Wire the fetcher** into the `SiteReplicationActor` ctor + its `Props` in `AkkaHostedService`.
|
||||
|
||||
**Step 6: Run tests — PASS. Commit** — `feat(site): replicate config by id + standby fetch (kills the intra-site frame trap)`
|
||||
|
||||
---
|
||||
|
||||
### Task 14: Retire the fat `DeployInstanceCommand` wire path
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** none (depends on Task 6, 7, 10, 11)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.Communication/CommunicationService.cs:116-135` (remove the now-unused `DeployInstanceAsync` Akka method — confirm no other caller)
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteCommunicationActor.cs:95-99` (remove the `Receive<DeployInstanceCommand>` forward — config no longer arrives that way)
|
||||
- Keep: `DeployInstanceCommand` record (still used **in-process** as the apply DTO inside `DeploymentManagerActor`).
|
||||
|
||||
**Step 1:** Grep for remaining references: `grep -rn "DeployInstanceAsync\|Receive<DeployInstanceCommand>" src/`. Remove the dead send + receive. Confirm `DeployInstanceCommand` is still referenced only inside `DeploymentManagerActor` (apply DTO) and tests.
|
||||
|
||||
**Step 2: Build the solution.** Expected: compiles with no unresolved references.
|
||||
|
||||
**Step 3: Commit** — `refactor(deploy): retire cross-cluster DeployInstanceCommand (config now fetched)`
|
||||
|
||||
---
|
||||
|
||||
### Task 15: appsettings + RUNBOOK
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Task 14
|
||||
|
||||
**Files:**
|
||||
- Modify central appsettings (add `Communication:CentralFetchBaseUrl`): `docker/central-node-a/appsettings.Central.json`, `docker/central-node-b/...`, `docker-env2/central-node-a/...`, `docker-env2/central-node-b/...`, `deploy/wonder-app-vd03/appsettings.Central.json`, and `src/ZB.MOM.WW.ScadaBridge.Host/appsettings.Central.json`
|
||||
- Modify: `deploy/wonder-app-vd03/RUNBOOK.md` (note the new setting + site→central HTTP reachability requirement)
|
||||
|
||||
**Step 1:** Set `CentralFetchBaseUrl` per environment:
|
||||
- docker: the Traefik LB URL reachable from site containers (e.g. `http://scadabridge-traefik:9000` or the central service name); verify against `docker/README.md` ports.
|
||||
- docker-env2: the env2 LB URL (91XX range).
|
||||
- wonder-app-vd03: the central management base URL the site service uses (single host → `http://localhost:8085` or the host address).
|
||||
|
||||
**Step 2:** RUNBOOK: under Upgrading, add "set `Communication:CentralFetchBaseUrl`; ensure site nodes can reach central's management port over HTTP(S)."
|
||||
|
||||
**Step 3: Commit** — `chore(deploy): CentralFetchBaseUrl appsettings + RUNBOOK`
|
||||
|
||||
---
|
||||
|
||||
### Task 16: Integration test — large config, replication, failover, supersession
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (depends on the full core: T1–T14)
|
||||
|
||||
**Files:**
|
||||
- Create: `tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/DeployNotifyFetchTests.cs` (follow the existing integration-test harness in that project)
|
||||
|
||||
**Step 1:** Tests:
|
||||
1. **>128 KB config deploys end-to-end** — stage a `PendingDeployment` whose `ConfigurationJson` is >150 KB; drive a deploy; assert the site applies it and central records Success (this is the exact case that fails today).
|
||||
2. **Standby gets it via fetch** — assert the standby's `deployed_configurations` row exists after the replicate-notify.
|
||||
3. **Supersession 404** — stage dep A then dep B for the same instance; assert `GET .../A/config` → 404 and `.../B/config` → 200.
|
||||
4. **Token** — wrong token → 401; expired row → 404.
|
||||
|
||||
**Step 2: Run — PASS. Commit** — `test(deploy): integration coverage for notify-and-fetch + supersession`
|
||||
|
||||
---
|
||||
|
||||
### Task 17: Live smoke on the docker cluster
|
||||
|
||||
**Classification:** small (manual/operational)
|
||||
**Estimated implement time:** ~5 min (+ build/redeploy wall time)
|
||||
**Parallelizable with:** none (depends on Task 16)
|
||||
|
||||
**Steps:**
|
||||
1. `bash docker/deploy.sh` (rebuild `scadabridge:latest`, recreate containers).
|
||||
2. Re-create the generic `DelmiaReceiver` composition scenario (or any instance whose flattened config exceeds 128 KB) and deploy via the CLI (`--username multi-role`).
|
||||
3. Assert: `deploy status` → Success; both site nodes hold the config (check each node's `deployed_configurations` or via debug snapshot); kill the active site node and confirm failover recovers the instance.
|
||||
4. Record results in the PR description. **No commit** unless smoke surfaces a fix.
|
||||
|
||||
---
|
||||
|
||||
### Task 18: FOLLOW-UP — standby/startup reconciliation (lower priority, sequence LAST)
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min (may split)
|
||||
**Parallelizable with:** none — implement only AFTER T1–T17 are green.
|
||||
|
||||
**Rationale:** Replication is best-effort with no retry and no startup reconciliation; a standby that is *down* during a deploy permanently misses that instance until its next deploy (pre-existing gap, independent of frame size). This makes replication self-healing.
|
||||
|
||||
**Auth decision (resolved):** NO static shared key. Reconciliation reuses the same trust model as deploy — the capability (fetch token) is delivered over the **trusted Akka ClusterClient** channel, the bulk config over HTTP. Each node, on startup, sends central its **local inventory** over Akka; central diffs it against the deployed snapshots and replies with **fresh fetch tokens only for the gap** (missing/stale instances). The node fetches the gap configs over the existing token-gated HTTP endpoint, which serves the re-staged `DeployedConfigSnapshot`. Central re-stages ONLY the gap (usually nothing), `stage-if-absent` so an in-flight deploy's pending row is never clobbered. Runs on **every** node (per-node, not the singleton) so a down standby self-heals. Fetch missing/stale; **log** orphans (never delete).
|
||||
|
||||
**Files:**
|
||||
- Create: Commons reconcile messages — `ReconcileSiteRequest(SiteIdentifier, NodeId, IReadOnlyDictionary<string,string> LocalNameToRevisionHash)` + `ReconcileSiteResponse(IReadOnlyList<ReconcileGapItem> Gap, IReadOnlyList<string> OrphanNames, string CentralFetchBaseUrl)` where `ReconcileGapItem(InstanceUniqueName, DeploymentId, RevisionHash, FetchToken)`.
|
||||
- Modify: `IDeploymentManagerRepository` / impl — add `GetExpectedDeploymentsForSiteAsync(siteId)` returning `(InstanceUniqueName, RevisionHash, DeploymentId)` from `DeployedConfigSnapshot`⋈`Instance` (by site); + a `StagePendingIfAbsentAsync` (insert-if-no-pending-row-for-instance, from the snapshot's config, fresh token + TTL) — does NOT supersede.
|
||||
- Modify: `CentralCommunicationActor` (or a small central reconcile handler) — handle `ReconcileSiteRequest`: compute expected set, diff vs reported inventory, `StagePendingIfAbsent` the gap with fresh tokens, reply `ReconcileSiteResponse`. Reuses the existing token + config-fetch endpoint.
|
||||
- Create: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/SiteReconciliationActor.cs` (per-node) — on startup (after local configs load), build the local inventory, Ask central via the site's central client, fetch the gap via `IDeploymentConfigFetcher`, guarded-write (`StoreDeployedConfigIfNewerAsync`), log orphans. Wire in `AkkaHostedService` per node + give it the central client + fetcher + storage.
|
||||
- Modify: site appsettings — add `ScadaBridge:Communication:CentralFetchBaseUrl` to the SITE files (the site now initiates fetches without a notify carrying it). NOTE: the response also carries `CentralFetchBaseUrl` from central, so site config is a fallback.
|
||||
- Test: central handler (gap diff + stage-if-absent + tokens), site reconcile (missing fetched, stale refreshed, orphan logged-not-deleted, no-gap = no fetch).
|
||||
|
||||
**Steps:** TDD; gate the gap fetch through the existing `IDeploymentConfigFetcher` + endpoint; one Akka round-trip; re-stage only the gap. Conservative: never delete orphans.
|
||||
|
||||
**Commit** — `feat(site): startup reconciliation of deployed configs (Akka inventory + gap fetch)`
|
||||
|
||||
---
|
||||
|
||||
### Task 19: Topology page — fast load (staleness off the live loop)
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (independent of T18; can run after)
|
||||
|
||||
**Rationale:** `Topology.razor` (`/deployment/topology`) already reads deployed state from central DB (it does NOT query sites). It's slow because the **staleness** badge loops over every deployed instance calling `DeploymentService.GetDeploymentComparisonAsync` → `FlattenAndValidateAsync` (a full re-flatten per instance) — on initial load AND again every 15 s via the live-updates timer.
|
||||
|
||||
**Files:**
|
||||
- Modify: `IDeploymentManagerRepository` / impl — add a bulk `GetDeployedSnapshotsBySiteAsync`/`GetAllDeployedSnapshotsAsync` (one query, avoids N snapshot lookups).
|
||||
- Modify: `src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Pages/Deployment/Topology.razor` — (a) load deployed state + the bulk snapshots fast; (b) take the staleness re-flatten OFF the 15 s live-update loop (live update refreshes only the cheap deployed state); (c) compute staleness once on initial load, **parallelized** across instances (not sequential awaits), and on an explicit "Re-check staleness" button.
|
||||
- Test: `tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/TopologyPageTests.cs` — assert the live-update path does NOT call `GetDeploymentComparisonAsync`; deployed state still renders; staleness computed on load + manual refresh.
|
||||
|
||||
**Steps:** TDD; keep the Stale/Current badge accurate (it only changes on edit/redeploy, so it doesn't belong on a 15 s poll); the deployed state (State + deployed-at) is the only thing the live timer refreshes.
|
||||
|
||||
**Commit** — `perf(ui): topology page — staleness off the live loop + bulk snapshot query`
|
||||
|
||||
---
|
||||
|
||||
## Final step
|
||||
|
||||
After all core tasks (T1–T17) are green: `dotnet build ZB.MOM.WW.ScadaBridge.slnx` (full solution) and run the affected test projects. Then **REQUIRED SUB-SKILL:** use superpowers-extended-cc:finishing-a-development-branch.
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"planPath": "docs/plans/2026-06-26-deploy-config-notify-and-fetch.md",
|
||||
"tasks": [
|
||||
{"id": 24, "subject": "Task 1: PendingDeployment entity + EF mapping + migration", "status": "completed"},
|
||||
{"id": 25, "subject": "Task 2: Pending-deployment repository (supersession + purge)", "status": "completed", "blockedBy": [24]},
|
||||
{"id": 26, "subject": "Task 3: RefreshDeploymentCommand message contract", "status": "completed"},
|
||||
{"id": 27, "subject": "Task 4: Fetch options + token helper", "status": "completed"},
|
||||
{"id": 28, "subject": "Task 5: Internal config-fetch endpoint (token-gated)", "status": "completed", "blockedBy": [24, 25, 27]},
|
||||
{"id": 29, "subject": "Task 6: Central send path — stage pending + send notify + promote", "status": "completed", "blockedBy": [25, 26, 27]},
|
||||
{"id": 30, "subject": "Task 7: CommunicationService.RefreshDeploymentAsync + routing", "status": "completed", "blockedBy": [26]},
|
||||
{"id": 31, "subject": "Task 8: Fix AskTimeoutException classification", "status": "completed", "blockedBy": [29]},
|
||||
{"id": 32, "subject": "Task 9: HTTP deployment-config fetcher + site DI + options", "status": "completed", "blockedBy": [27]},
|
||||
{"id": 33, "subject": "Task 10: DeploymentManagerActor handles RefreshDeploymentCommand", "status": "completed", "blockedBy": [26, 32]},
|
||||
{"id": 34, "subject": "Task 11: SiteCommunicationActor routes RefreshDeploymentCommand", "status": "completed", "blockedBy": [26]},
|
||||
{"id": 35, "subject": "Task 12: Unify replication on notify-and-fetch (id-only + standby fetch)", "status": "completed", "blockedBy": [32, 36]},
|
||||
{"id": 36, "subject": "Task 13: Site storage older-write guard", "status": "completed"},
|
||||
{"id": 37, "subject": "Task 14: Retire fat DeployInstanceCommand wire path", "status": "completed", "blockedBy": [29, 30, 33, 34]},
|
||||
{"id": 38, "subject": "Task 15: appsettings CentralFetchBaseUrl + RUNBOOK", "status": "completed"},
|
||||
{"id": 39, "subject": "Task 16: Integration test — large config, supersession, token", "status": "completed", "blockedBy": [29, 33, 35, 37]},
|
||||
{"id": 40, "subject": "Task 17: Live smoke on docker cluster (found+fixed 2 reconcile bugs)", "status": "completed", "blockedBy": [42, 43, 44]},
|
||||
{"id": 41, "subject": "Task 18a: reconcile messages + repo (expected-set + stage-if-absent)", "status": "completed", "blockedBy": [39]},
|
||||
{"id": 42, "subject": "Task 18b: central reconcile handler", "status": "completed", "blockedBy": [41]},
|
||||
{"id": 43, "subject": "Task 18c: site reconciliation actor (per-node) + wiring", "status": "completed", "blockedBy": [41]},
|
||||
{"id": 44, "subject": "Task 19: Topology page fast load (staleness off live loop)", "status": "completed"}
|
||||
],
|
||||
"lastUpdated": "2026-06-26"
|
||||
}
|
||||
@@ -9,6 +9,8 @@
|
||||
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums
|
||||
@using ZB.MOM.WW.ScadaBridge.DeploymentManager
|
||||
@using ZB.MOM.WW.ScadaBridge.TemplateEngine.Services
|
||||
@using System.Collections.Concurrent
|
||||
@using Microsoft.Extensions.DependencyInjection
|
||||
@attribute [Authorize(Policy = AuthorizationPolicies.RequireDeployment)]
|
||||
@inject ITemplateEngineRepository TemplateEngineRepository
|
||||
@inject ISiteRepository SiteRepository
|
||||
@@ -21,6 +23,7 @@
|
||||
@inject NavigationManager NavigationManager
|
||||
@inject IJSRuntime JSRuntime
|
||||
@inject IDialogService Dialog
|
||||
@inject IServiceScopeFactory ScopeFactory
|
||||
@implements IDisposable
|
||||
|
||||
<div class="container-fluid mt-3">
|
||||
@@ -77,6 +80,19 @@
|
||||
<button class="btn btn-outline-secondary" aria-label="Expand all areas" @onclick="() => _tree?.ExpandAll()">Expand</button>
|
||||
<button class="btn btn-outline-secondary" aria-label="Collapse all areas" @onclick="() => _tree?.CollapseAll()">Collapse</button>
|
||||
</div>
|
||||
<button class="btn btn-outline-secondary btn-sm" aria-label="Re-check staleness"
|
||||
title="Recompute the Stale/Current badge for every deployed instance"
|
||||
@onclick="RecheckStalenessAsync" disabled="@_recheckingStaleness">
|
||||
@if (_recheckingStaleness)
|
||||
{
|
||||
<span class="spinner-border spinner-border-sm me-1" role="status" aria-hidden="true"></span>
|
||||
<span>Checking…</span>
|
||||
}
|
||||
else
|
||||
{
|
||||
<span><i class="bi bi-arrow-repeat me-1"></i>Re-check staleness</span>
|
||||
}
|
||||
</button>
|
||||
<div class="form-check form-switch ms-2 mb-0">
|
||||
<input type="checkbox" class="form-check-input" id="live-updates"
|
||||
checked="@_liveUpdates" @onchange="OnLiveUpdatesToggled" />
|
||||
@@ -121,9 +137,18 @@
|
||||
private List<Area> _allAreas = new();
|
||||
private Dictionary<int, bool> _stalenessMap = new();
|
||||
|
||||
/// <summary>
|
||||
/// Max concurrent per-instance staleness comparisons. Each comparison
|
||||
/// re-flattens the instance's config, so the fan-out is capped to avoid a
|
||||
/// thundering herd of flattens on a large topology (mirrors the
|
||||
/// AlarmSummaryService DebugViewSnapshot fan-out cap).
|
||||
/// </summary>
|
||||
private const int MaxConcurrentStalenessChecks = 8;
|
||||
|
||||
private bool _loading = true;
|
||||
private string? _errorMessage;
|
||||
private bool _actionInProgress;
|
||||
private bool _recheckingStaleness;
|
||||
|
||||
private string _searchText = string.Empty;
|
||||
|
||||
@@ -178,7 +203,11 @@
|
||||
InvokeAsync(async () =>
|
||||
{
|
||||
if (!_liveUpdates) return;
|
||||
await LoadDataAsync();
|
||||
// Staleness is deliberately OFF the live loop: it only changes when a
|
||||
// template/instance is edited or redeployed, and recomputing it (a
|
||||
// full re-flatten per deployed instance) every 15s is the page's
|
||||
// dominant cost. The poll refreshes only the cheap deployed state.
|
||||
await LoadDeployedStateAsync();
|
||||
StateHasChanged();
|
||||
});
|
||||
}, null, LiveUpdatesInterval, LiveUpdatesInterval);
|
||||
@@ -220,7 +249,25 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Full reload: the cheap deployed state plus a one-shot staleness recompute.
|
||||
// Used on initial load, the manual "Refresh" button, and after any mutating
|
||||
// action (deploy/enable/disable/delete/move/rename) — all of which can change
|
||||
// staleness. The live-update timer deliberately does NOT call this; it calls
|
||||
// LoadDeployedStateAsync alone (staleness is off the 15s poll).
|
||||
private async Task LoadDataAsync()
|
||||
{
|
||||
await LoadDeployedStateAsync();
|
||||
await ComputeStalenessAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fast path — loads the cheap deployed state (sites, instances, areas →
|
||||
/// hierarchy + State badge) WITHOUT recomputing per-instance staleness. These
|
||||
/// are all single, indexed central-DB reads. This is exactly what the
|
||||
/// live-update timer polls every 15s. Internal so the live-update regression
|
||||
/// test can drive the timer's code path directly.
|
||||
/// </summary>
|
||||
internal async Task LoadDeployedStateAsync()
|
||||
{
|
||||
_loading = true;
|
||||
_errorMessage = null;
|
||||
@@ -242,20 +289,6 @@
|
||||
_allAreas.AddRange(areas);
|
||||
}
|
||||
|
||||
_stalenessMap.Clear();
|
||||
foreach (var inst in _allInstances.Where(i => i.State != InstanceState.NotDeployed))
|
||||
{
|
||||
try
|
||||
{
|
||||
var comparison = await DeploymentService.GetDeploymentComparisonAsync(inst.Id);
|
||||
_stalenessMap[inst.Id] = comparison.IsSuccess && comparison.Value.IsStale;
|
||||
}
|
||||
catch
|
||||
{
|
||||
_stalenessMap[inst.Id] = false;
|
||||
}
|
||||
}
|
||||
|
||||
BuildTree();
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -265,6 +298,81 @@
|
||||
_loading = false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expensive path — recomputes the Stale/Current badge for every deployed
|
||||
/// instance via <c>DeploymentService.GetDeploymentComparisonAsync</c>, which
|
||||
/// re-flattens each instance's current config and compares it to the deployed
|
||||
/// snapshot. This is the page's dominant cost, so it runs only on initial
|
||||
/// load and on demand (the "Re-check staleness" button) — never on the 15s
|
||||
/// live-update loop. The per-instance comparisons fan out with bounded
|
||||
/// concurrency (<see cref="MaxConcurrentStalenessChecks"/>), each on its own
|
||||
/// DI scope so the parallel flattens never share the circuit-scoped DbContext
|
||||
/// (mirrors KpiHistoryQueryService's per-query scope).
|
||||
/// </summary>
|
||||
private async Task ComputeStalenessAsync()
|
||||
{
|
||||
var deployedIds = _allInstances
|
||||
.Where(i => i.State != InstanceState.NotDeployed)
|
||||
.Select(i => i.Id)
|
||||
.ToList();
|
||||
|
||||
if (deployedIds.Count == 0)
|
||||
{
|
||||
_stalenessMap = new Dictionary<int, bool>();
|
||||
BuildTree();
|
||||
return;
|
||||
}
|
||||
|
||||
var results = new ConcurrentDictionary<int, bool>();
|
||||
using var gate = new SemaphoreSlim(MaxConcurrentStalenessChecks, MaxConcurrentStalenessChecks);
|
||||
|
||||
await Task.WhenAll(deployedIds.Select(id => ComputeOneStalenessAsync(id, gate, results)));
|
||||
|
||||
_stalenessMap = new Dictionary<int, bool>(results);
|
||||
BuildTree();
|
||||
}
|
||||
|
||||
private async Task ComputeOneStalenessAsync(
|
||||
int instanceId, SemaphoreSlim gate, ConcurrentDictionary<int, bool> results)
|
||||
{
|
||||
await gate.WaitAsync();
|
||||
try
|
||||
{
|
||||
// Fresh DI scope per comparison: GetDeploymentComparisonAsync re-flattens
|
||||
// through the scoped DbContext, which must not be shared across the
|
||||
// concurrent fan-out. Each scope gets its own DbContext.
|
||||
await using var scope = ScopeFactory.CreateAsyncScope();
|
||||
var deploymentService = scope.ServiceProvider.GetRequiredService<DeploymentService>();
|
||||
var comparison = await deploymentService.GetDeploymentComparisonAsync(instanceId);
|
||||
results[instanceId] = comparison.IsSuccess && comparison.Value.IsStale;
|
||||
}
|
||||
catch
|
||||
{
|
||||
results[instanceId] = false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
// Manual "Re-check staleness" toolbar action. Shows a spinner while the
|
||||
// bounded-parallel one-shot runs, then refreshes the Stale/Current badges.
|
||||
private async Task RecheckStalenessAsync()
|
||||
{
|
||||
if (_recheckingStaleness) return;
|
||||
_recheckingStaleness = true;
|
||||
StateHasChanged();
|
||||
try
|
||||
{
|
||||
await ComputeStalenessAsync();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_recheckingStaleness = false;
|
||||
}
|
||||
}
|
||||
|
||||
private void OnSearchChanged()
|
||||
{
|
||||
BuildTree();
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// A flattened instance config staged for a single in-flight deployment, fetched
|
||||
/// by the site over HTTP instead of being shipped inside an Akka message (avoids
|
||||
/// the 128 KB frame limit). Keyed by DeploymentId; at most one row per InstanceId
|
||||
/// (a newer deploy supersedes the prior pending row). Carries a per-deployment
|
||||
/// fetch token and a TTL. Promoted to DeployedConfigSnapshot on success; purged
|
||||
/// on failure/timeout/TTL.
|
||||
/// </summary>
|
||||
public class PendingDeployment
|
||||
{
|
||||
/// <summary>Primary key.</summary>
|
||||
public int Id { get; set; }
|
||||
|
||||
/// <summary>Unique deployment identifier assigned at deploy time; the fetch key for the staged config.</summary>
|
||||
public string DeploymentId { get; set; }
|
||||
|
||||
/// <summary>Foreign key to the owning <c>Instance</c> entity. At most one pending row per instance.</summary>
|
||||
public int InstanceId { get; set; }
|
||||
|
||||
/// <summary>Revision hash of the flattened configuration, used for staleness detection.</summary>
|
||||
public string RevisionHash { get; set; }
|
||||
|
||||
/// <summary>JSON-serialized flattened configuration staged for the site to fetch.</summary>
|
||||
public string ConfigurationJson { get; set; }
|
||||
|
||||
/// <summary>Per-deployment fetch token presented by the site to authorize the HTTP fetch.</summary>
|
||||
public string Token { get; set; }
|
||||
|
||||
/// <summary>UTC timestamp when this pending row was created.</summary>
|
||||
public DateTimeOffset CreatedAtUtc { get; set; }
|
||||
|
||||
/// <summary>UTC timestamp after which this pending row is considered expired (TTL) and may be purged.</summary>
|
||||
public DateTimeOffset ExpiresAtUtc { get; set; }
|
||||
|
||||
/// <summary>Initializes a new pending deployment with its identity, staged configuration, fetch token, and TTL.</summary>
|
||||
/// <param name="deploymentId">Unique deployment identifier (fetch key).</param>
|
||||
/// <param name="instanceId">Owning instance identifier.</param>
|
||||
/// <param name="revisionHash">Revision hash of the flattened configuration.</param>
|
||||
/// <param name="configurationJson">JSON-serialized flattened configuration.</param>
|
||||
/// <param name="token">Per-deployment fetch token.</param>
|
||||
/// <param name="createdAtUtc">UTC creation timestamp.</param>
|
||||
/// <param name="expiresAtUtc">UTC expiry timestamp (TTL).</param>
|
||||
public PendingDeployment(
|
||||
string deploymentId, int instanceId, string revisionHash,
|
||||
string configurationJson, string token,
|
||||
DateTimeOffset createdAtUtc, DateTimeOffset expiresAtUtc)
|
||||
{
|
||||
DeploymentId = deploymentId ?? throw new ArgumentNullException(nameof(deploymentId));
|
||||
InstanceId = instanceId;
|
||||
RevisionHash = revisionHash ?? throw new ArgumentNullException(nameof(revisionHash));
|
||||
ConfigurationJson = configurationJson ?? throw new ArgumentNullException(nameof(configurationJson));
|
||||
Token = token ?? throw new ArgumentNullException(nameof(token));
|
||||
CreatedAtUtc = createdAtUtc;
|
||||
ExpiresAtUtc = expiresAtUtc;
|
||||
}
|
||||
}
|
||||
+92
@@ -1,5 +1,6 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
@@ -136,6 +137,97 @@ public interface IDeploymentManagerRepository
|
||||
/// <returns>A task representing the asynchronous operation.</returns>
|
||||
Task DeleteDeployedSnapshotAsync(int instanceId, CancellationToken cancellationToken = default);
|
||||
|
||||
// Notify-and-fetch: PendingDeployment staging store
|
||||
/// <summary>
|
||||
/// Stages a flattened instance config for an in-flight deployment. Supersession:
|
||||
/// any prior pending row for the same <c>InstanceId</c> is removed first, so at most
|
||||
/// one pending row exists per instance (safe because the per-instance operation lock
|
||||
/// serializes same-instance deploys). Does NOT call <see cref="SaveChangesAsync"/> —
|
||||
/// the caller commits, mirroring the other Add/Delete repository methods.
|
||||
/// </summary>
|
||||
/// <param name="pending">The pending deployment to stage.</param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>A task representing the asynchronous operation.</returns>
|
||||
Task AddPendingDeploymentAsync(PendingDeployment pending, CancellationToken cancellationToken = default);
|
||||
/// <summary>
|
||||
/// Gets a pending deployment by its deployment ID (the fetch key).
|
||||
/// </summary>
|
||||
/// <param name="deploymentId">The deployment ID.</param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>The pending deployment, or null if not found.</returns>
|
||||
Task<PendingDeployment?> GetPendingDeploymentByIdAsync(string deploymentId, CancellationToken cancellationToken = default);
|
||||
/// <summary>
|
||||
/// Gets the pending deployment staged for a given instance (there is at most one per instance
|
||||
/// by design — <see cref="AddPendingDeploymentAsync"/> supersedes and
|
||||
/// <see cref="StagePendingIfAbsentAsync"/> inserts only if absent). Used by startup reconcile:
|
||||
/// when <see cref="StagePendingIfAbsentAsync"/> reports a row already exists (a concurrent
|
||||
/// reconcile from the other node, or an in-flight deploy), the handler reads that existing row
|
||||
/// so a second concurrently-missing node can fetch the same pending config with its (multi-use,
|
||||
/// TTL-bound) token instead of being omitted from the gap. Defensive against >1 row: returns
|
||||
/// the most recently created one. When <paramref name="nowUtc"/> is supplied, EXPIRED rows
|
||||
/// (<c>ExpiresAtUtc <= nowUtc</c>) are filtered out so the caller never receives a token the
|
||||
/// config-fetch endpoint would 404.
|
||||
/// </summary>
|
||||
/// <param name="instanceId">The instance whose pending deployment to read.</param>
|
||||
/// <param name="nowUtc">When supplied, only rows with <c>ExpiresAtUtc > nowUtc</c> are considered (expired rows excluded).</param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>The pending deployment for the instance, or null if none is staged.</returns>
|
||||
Task<PendingDeployment?> GetPendingDeploymentByInstanceIdAsync(int instanceId, DateTimeOffset? nowUtc = null, CancellationToken cancellationToken = default);
|
||||
/// <summary>
|
||||
/// Deletes a pending deployment by its deployment ID. No-op if not found. Does NOT
|
||||
/// call <see cref="SaveChangesAsync"/> — the caller commits, mirroring the other
|
||||
/// Add/Delete repository methods.
|
||||
/// </summary>
|
||||
/// <param name="deploymentId">The deployment ID to delete.</param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>A task representing the asynchronous operation.</returns>
|
||||
Task DeletePendingDeploymentByIdAsync(string deploymentId, CancellationToken cancellationToken = default);
|
||||
/// <summary>
|
||||
/// Purges all pending deployments whose <c>ExpiresAtUtc</c> is at or before
|
||||
/// <paramref name="nowUtc"/> (TTL maintenance). This is a self-contained maintenance
|
||||
/// operation: it commits its own delete and returns the number of rows removed.
|
||||
/// </summary>
|
||||
/// <param name="nowUtc">The current UTC time; rows with <c>ExpiresAtUtc <= nowUtc</c> are purged.</param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>The number of pending deployments purged.</returns>
|
||||
Task<int> PurgeExpiredPendingDeploymentsAsync(DateTimeOffset nowUtc, CancellationToken cancellationToken = default);
|
||||
|
||||
// Startup reconciliation: expected-set query and insert-if-absent staging
|
||||
|
||||
/// <summary>
|
||||
/// Returns the set of instances that central considers deployed for the given site — the join of
|
||||
/// <c>DeployedConfigSnapshot</c> (by InstanceId) with <c>Instance</c> (where SiteId == siteId).
|
||||
/// Instances without a snapshot are excluded. No <c>ConfigurationJson</c> is loaded; the full
|
||||
/// config is fetched on demand via the HTTP endpoint using a freshly-minted token from
|
||||
/// <see cref="StagePendingIfAbsentAsync"/>.
|
||||
/// </summary>
|
||||
/// <param name="siteId">The site primary key to filter on.</param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>All deployed instances for the site, as lightweight <see cref="ExpectedDeployment"/> projections.</returns>
|
||||
Task<IReadOnlyList<ExpectedDeployment>> GetExpectedDeploymentsForSiteAsync(int siteId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a <see cref="PendingDeployment"/> for the instance ONLY IF no pending row already
|
||||
/// exists for that <c>InstanceId</c>. An existing pending row signals an in-flight deploy that
|
||||
/// is already delivering to the node — do NOT supersede it (contrast with
|
||||
/// <see cref="AddPendingDeploymentAsync"/>, which supersedes). Returns <c>true</c> if the row
|
||||
/// was staged, <c>false</c> if an existing row was found and left untouched.
|
||||
/// This method is self-contained: it commits its own save and returns a meaningful result,
|
||||
/// matching the convention of <see cref="PurgeExpiredPendingDeploymentsAsync"/>.
|
||||
/// </summary>
|
||||
/// <param name="instanceId">The instance to stage the pending deployment for.</param>
|
||||
/// <param name="deploymentId">The deployment ID (fetch key).</param>
|
||||
/// <param name="revisionHash">Revision hash of the flattened configuration.</param>
|
||||
/// <param name="configurationJson">JSON-serialized flattened configuration.</param>
|
||||
/// <param name="token">Short-TTL fetch token the site node will present to the HTTP endpoint.</param>
|
||||
/// <param name="createdAtUtc">UTC timestamp for this pending row.</param>
|
||||
/// <param name="expiresAtUtc">UTC expiry after which the token is no longer valid.</param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns><c>true</c> if staged; <c>false</c> if a pending row already existed and was left unchanged.</returns>
|
||||
Task<bool> StagePendingIfAbsentAsync(int instanceId, string deploymentId, string revisionHash,
|
||||
string configurationJson, string token, DateTimeOffset createdAtUtc, DateTimeOffset expiresAtUtc,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
// Instance lookups for deployment pipeline
|
||||
/// <summary>
|
||||
/// Gets an instance by ID.
|
||||
|
||||
@@ -1,9 +1,22 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// Internal site-side apply DTO for a single instance deployment. (Its cross-cluster wire
|
||||
/// use is being retired; notify-and-fetch now moves only ids over the wire.)
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <paramref name="CentralFetchBaseUrl"/> and <paramref name="FetchToken"/> are optional and
|
||||
/// carry the central fetch coordinates down the in-process apply path so that
|
||||
/// <c>ApplyDeployment</c> can replicate an id-only notify-and-fetch message to the standby
|
||||
/// node (instead of the full config JSON, which tripped the 128 KB intra-site frame trap).
|
||||
/// They are populated on the notify-and-fetch path; all other construction sites omit them.
|
||||
/// </remarks>
|
||||
public record DeployInstanceCommand(
|
||||
string DeploymentId,
|
||||
string InstanceUniqueName,
|
||||
string RevisionHash,
|
||||
string FlattenedConfigurationJson,
|
||||
string DeployedBy,
|
||||
DateTimeOffset Timestamp);
|
||||
DateTimeOffset Timestamp,
|
||||
string? CentralFetchBaseUrl = null,
|
||||
string? FetchToken = null);
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// Site→central (over ClusterClient) on node startup: the node's local deployed inventory,
|
||||
/// so central can reply with fetch tokens for whatever the node is missing or stale
|
||||
/// (self-heal a node that was down during a deploy).
|
||||
/// </summary>
|
||||
public record ReconcileSiteRequest(
|
||||
string SiteIdentifier,
|
||||
string NodeId,
|
||||
IReadOnlyDictionary<string, string> LocalNameToRevisionHash);
|
||||
@@ -0,0 +1,21 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// Central→site reply: fresh fetch tokens for the gap (instances the node is missing or has at
|
||||
/// a stale revision), orphan names to log (present locally but no longer deployed centrally),
|
||||
/// and the base URL to fetch from.
|
||||
/// </summary>
|
||||
public record ReconcileSiteResponse(
|
||||
IReadOnlyList<ReconcileGapItem> Gap,
|
||||
IReadOnlyList<string> OrphanNames,
|
||||
string CentralFetchBaseUrl);
|
||||
|
||||
/// <summary>
|
||||
/// One instance the node must (re)fetch, with a freshly-minted short-TTL token.
|
||||
/// </summary>
|
||||
public record ReconcileGapItem(
|
||||
string InstanceUniqueName,
|
||||
string DeploymentId,
|
||||
string RevisionHash,
|
||||
bool IsEnabled,
|
||||
string FetchToken);
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// Small central→site notify (replaces the fat DeployInstanceCommand on the wire).
|
||||
/// Routes via the deployment-manager singleton proxy to the active node, which
|
||||
/// fetches the flattened config from <see cref="CentralFetchBaseUrl"/> using
|
||||
/// <see cref="FetchToken"/> and applies it. The reply is the existing
|
||||
/// DeploymentStatusResponse.
|
||||
/// </summary>
|
||||
public record RefreshDeploymentCommand(
|
||||
string DeploymentId,
|
||||
string InstanceUniqueName,
|
||||
string RevisionHash,
|
||||
string DeployedBy,
|
||||
DateTimeOffset Timestamp,
|
||||
string CentralFetchBaseUrl,
|
||||
string FetchToken);
|
||||
@@ -0,0 +1,22 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// Generates and compares the single-purpose, short-TTL token that authorizes a
|
||||
/// site's HTTP fetch of one deployment's flattened config. URL-safe; compared in
|
||||
/// constant time to avoid timing oracles.
|
||||
/// </summary>
|
||||
public static class DeploymentFetchToken
|
||||
{
|
||||
/// <summary>Generates a URL-safe random token (256 bits of entropy).</summary>
|
||||
public static string Generate() =>
|
||||
Convert.ToBase64String(RandomNumberGenerator.GetBytes(32))
|
||||
.Replace('+', '-').Replace('/', '_').TrimEnd('=');
|
||||
|
||||
/// <summary>Constant-time string comparison (no early-out on first mismatch).</summary>
|
||||
public static bool ConstantTimeEquals(string a, string b) =>
|
||||
CryptographicOperations.FixedTimeEquals(
|
||||
Encoding.UTF8.GetBytes(a), Encoding.UTF8.GetBytes(b));
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// Lightweight projection of an instance's last-deployed state, used by the startup-reconciliation
|
||||
/// path to compute the gap between central's expected deployed set and a site node's local inventory.
|
||||
/// No <c>ConfigurationJson</c> is included — the full config is only fetched for gap instances via
|
||||
/// the HTTP endpoint using the freshly-minted <see cref="ReconcileGapItem.FetchToken"/>.
|
||||
/// </summary>
|
||||
/// <param name="InstanceId">The instance primary key.</param>
|
||||
/// <param name="InstanceUniqueName">The system-wide unique name for the instance.</param>
|
||||
/// <param name="RevisionHash">Revision hash of the flattened configuration at last successful deploy.</param>
|
||||
/// <param name="DeploymentId">Deployment ID of the last successful deploy (the fetch key).</param>
|
||||
/// <param name="IsEnabled">True when the instance's current <c>State</c> is <c>Enabled</c>.</param>
|
||||
public record ExpectedDeployment(
|
||||
int InstanceId,
|
||||
string InstanceUniqueName,
|
||||
string RevisionHash,
|
||||
string DeploymentId,
|
||||
bool IsEnabled);
|
||||
@@ -7,6 +7,7 @@ using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
@@ -213,6 +214,12 @@ public class CentralCommunicationActor : ReceiveActor
|
||||
// Audit Log (#23 M3) combined-telemetry ingest: routes to the same proxy
|
||||
// the same way; the proxy replies with an IngestCachedTelemetryReply.
|
||||
Receive<IngestCachedTelemetryCommand>(HandleIngestCachedTelemetry);
|
||||
|
||||
// Startup reconciliation: a site node forwards its local deployed inventory on
|
||||
// startup via ClusterClient. Resolve the scoped ReconcileService, diff the
|
||||
// inventory against central's expected set, and pipe the ReconcileSiteResponse
|
||||
// (gap fetch tokens + orphans) straight back to the site node's ClusterClient.
|
||||
Receive<ReconcileSiteRequest>(HandleReconcileSiteRequest);
|
||||
}
|
||||
|
||||
private void HandleNotificationSubmit(NotificationSubmit msg)
|
||||
@@ -297,6 +304,48 @@ public class CentralCommunicationActor : ReceiveActor
|
||||
.PipeTo(replyTo);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Startup reconciliation (site→central over ClusterClient): resolve the scoped
|
||||
/// <see cref="ReconcileService"/> in a DI scope, diff the node's reported inventory
|
||||
/// against central's expected set, and pipe the <see cref="ReconcileSiteResponse"/>
|
||||
/// back to the site node's ClusterClient path. The actor stays thin — all the diff
|
||||
/// and staging logic lives in the service. Mirrors the DB-access pattern used by
|
||||
/// <see cref="LoadSiteAddressesFromDb"/> (Task.Run + CreateScope + PipeTo) and the
|
||||
/// Sender-preservation pattern of <see cref="HandleIngestAuditEvents"/>.
|
||||
///
|
||||
/// On a faulted task PipeTo delivers a <see cref="Status.Failure"/> to the node; its
|
||||
/// Ask faults and it simply retries reconcile on the next startup — reconcile is
|
||||
/// best-effort, so the fault is allowed to propagate rather than being swallowed.
|
||||
/// </summary>
|
||||
private void HandleReconcileSiteRequest(ReconcileSiteRequest msg)
|
||||
{
|
||||
// Capture Sender before the async/PipeTo — Akka resets Sender between dispatches.
|
||||
var replyTo = Sender;
|
||||
|
||||
// Bound the DB work by the actor lifecycle (Communication-019). The CTS may have
|
||||
// been disposed by PostStop on a racing late message; treat that as "actor gone".
|
||||
CancellationToken ct;
|
||||
try
|
||||
{
|
||||
ct = _lifecycleCts.Token;
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_log.Debug(
|
||||
"Handling ReconcileSiteRequest from site {0} node {1} ({2} local instance(s))",
|
||||
msg.SiteIdentifier, msg.NodeId, msg.LocalNameToRevisionHash.Count);
|
||||
|
||||
Task.Run(async () =>
|
||||
{
|
||||
using var scope = _serviceProvider.CreateScope();
|
||||
var service = scope.ServiceProvider.GetRequiredService<ReconcileService>();
|
||||
return await service.ReconcileAsync(msg, ct).ConfigureAwait(false);
|
||||
}).PipeTo(replyTo);
|
||||
}
|
||||
|
||||
private void HandleHeartbeat(HeartbeatMessage heartbeat)
|
||||
{
|
||||
var aggregator = _serviceProvider.GetService<ICentralHealthAggregator>();
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Central cluster singleton that periodically reclaims expired (TTL-elapsed)
|
||||
/// <c>PendingDeployment</c> staging rows from the central configuration database —
|
||||
/// the notify-and-fetch deploy transport stages each deploy's flattened config in a
|
||||
/// <c>PendingDeployment</c> row that the site fetches over HTTP using a per-deployment
|
||||
/// token, and this actor is the maintenance cadence that sweeps rows whose TTL has
|
||||
/// elapsed without a re-deploy superseding them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Best-effort, not readiness-gated.</b> This purge is pure hygiene: supersession
|
||||
/// already bounds the table to ≤1 pending row per instance, and the config-fetch
|
||||
/// endpoint enforces the TTL on read (expired rows return 404), so correctness never
|
||||
/// depends on this purge running. It exists only to reclaim rows left behind by
|
||||
/// instances that are deployed once and never re-deployed. Like
|
||||
/// <c>KpiHistoryRecorderActor</c>, it is deliberately absent from
|
||||
/// <c>RequiredSingletonsHealthCheck</c> — it must never gate <c>/health/ready</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Continue-on-error.</b> The tick handler swallows every exception and logs it; a
|
||||
/// transient SQL failure on one tick must not crash the singleton — the next tick
|
||||
/// retries. The per-tick try/catch (not the supervisor) is what keeps the actor alive.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>DI scopes.</b> <see cref="IDeploymentManagerRepository"/> is a scoped EF Core
|
||||
/// service registered by <c>AddConfigurationDatabase</c>. The singleton opens one DI
|
||||
/// scope per tick and resolves the repository there, mirroring the
|
||||
/// <c>AuditLogPurgeActor</c> / <c>KpiHistoryRecorderActor</c> pattern.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class PendingDeploymentPurgeActor : ReceiveActor
|
||||
{
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly TimeSpan _interval;
|
||||
private readonly ILogger<PendingDeploymentPurgeActor> _logger;
|
||||
private ICancelable? _timer;
|
||||
|
||||
/// <summary>Initializes a new instance of <see cref="PendingDeploymentPurgeActor"/> and registers the tick handler.</summary>
|
||||
/// <param name="services">Root DI provider used to create a scoped repository per tick.</param>
|
||||
/// <param name="options">Communication options supplying the purge interval.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
public PendingDeploymentPurgeActor(
|
||||
IServiceProvider services,
|
||||
IOptions<CommunicationOptions> options,
|
||||
ILogger<PendingDeploymentPurgeActor> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
_services = services;
|
||||
_interval = options.Value.PendingDeploymentPurgeInterval;
|
||||
_logger = logger;
|
||||
|
||||
ReceiveAsync<PurgeTick>(_ => OnTickAsync());
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
base.PreStart();
|
||||
_timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(
|
||||
initialDelay: _interval,
|
||||
interval: _interval,
|
||||
receiver: Self,
|
||||
message: PurgeTick.Instance,
|
||||
sender: Self);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PostStop()
|
||||
{
|
||||
_timer?.Cancel();
|
||||
base.PostStop();
|
||||
}
|
||||
|
||||
private async Task OnTickAsync()
|
||||
{
|
||||
// CreateAsyncScope + await using so the scoped EF Core DbContext (IAsyncDisposable)
|
||||
// disposes asynchronously without blocking on connection cleanup.
|
||||
await using var scope = _services.CreateAsyncScope();
|
||||
IDeploymentManagerRepository repository;
|
||||
try
|
||||
{
|
||||
repository = scope.ServiceProvider.GetRequiredService<IDeploymentManagerRepository>();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to resolve IDeploymentManagerRepository for PendingDeployment purge tick.");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var purged = await repository
|
||||
.PurgeExpiredPendingDeploymentsAsync(DateTimeOffset.UtcNow)
|
||||
.ConfigureAwait(false);
|
||||
if (purged > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Purged {Count} expired PendingDeployment staging row(s).", purged);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Best-effort: a failed tick must not crash the singleton. The next
|
||||
// interval retries; correctness does not depend on any single tick.
|
||||
_logger.LogError(ex, "PendingDeployment purge tick failed; retrying on the next interval.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Self-tick triggering one expired-row purge pass.</summary>
|
||||
internal sealed class PurgeTick
|
||||
{
|
||||
public static readonly PurgeTick Instance = new();
|
||||
private PurgeTick() { }
|
||||
}
|
||||
}
|
||||
@@ -92,9 +92,9 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
Receive<RegisterLocalHandler>(HandleRegisterLocalHandler);
|
||||
|
||||
// Pattern 1: Instance Deployment — forward to Deployment Manager
|
||||
Receive<DeployInstanceCommand>(msg =>
|
||||
Receive<RefreshDeploymentCommand>(msg =>
|
||||
{
|
||||
_log.Debug("Routing DeployInstanceCommand for {0} to DeploymentManager", msg.InstanceUniqueName);
|
||||
_log.Debug("Routing RefreshDeploymentCommand for {0} to DeploymentManager", msg.InstanceUniqueName);
|
||||
_deploymentManagerProxy.Forward(msg);
|
||||
});
|
||||
|
||||
@@ -359,6 +359,34 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
new ClusterClient.Send("/user/central-communication", msg), Sender);
|
||||
});
|
||||
|
||||
// Site startup reconciliation (Task 18): forward the node's local-inventory
|
||||
// ReconcileSiteRequest to the central cluster. The original Sender (the
|
||||
// SiteReconciliationActor's Ask) is passed as the ClusterClient.Send sender so
|
||||
// the ReconcileSiteResponse routes straight back to the waiting Ask, not here.
|
||||
// Mirrors IngestAuditEventsCommand.
|
||||
Receive<ReconcileSiteRequest>(msg =>
|
||||
{
|
||||
if (_centralClient == null)
|
||||
{
|
||||
// No ClusterClient registered yet (e.g. central contact points not
|
||||
// configured, or registration not yet completed). Faulting the Ask makes
|
||||
// the SiteReconciliationActor treat the pass as best-effort-failed; it
|
||||
// logs a warning and retries reconcile on the next node startup.
|
||||
_log.Warning(
|
||||
"Cannot forward ReconcileSiteRequest for site {0} node {1} — no central ClusterClient registered",
|
||||
msg.SiteIdentifier, msg.NodeId);
|
||||
Sender.Tell(new Status.Failure(
|
||||
new InvalidOperationException("Central ClusterClient not registered")));
|
||||
return;
|
||||
}
|
||||
|
||||
_log.Debug(
|
||||
"Forwarding ReconcileSiteRequest for site {0} node {1} ({2} local instance(s)) to central",
|
||||
msg.SiteIdentifier, msg.NodeId, msg.LocalNameToRevisionHash.Count);
|
||||
_centralClient.Tell(
|
||||
new ClusterClient.Send("/user/central-communication", msg), Sender);
|
||||
});
|
||||
|
||||
// Internal: send heartbeat tick
|
||||
Receive<SendHeartbeat>(_ => SendHeartbeatToCentral());
|
||||
|
||||
|
||||
@@ -58,4 +58,26 @@ public class CommunicationOptions
|
||||
|
||||
/// <summary>Akka.Remote transport failure detection threshold.</summary>
|
||||
public TimeSpan TransportFailureThreshold { get; set; } = TimeSpan.FromSeconds(15);
|
||||
|
||||
/// <summary>
|
||||
/// Base URL (Traefik/LB) the SITE uses to fetch deploy configs from central,
|
||||
/// e.g. "https://central.example:9000". Carried in RefreshDeploymentCommand so
|
||||
/// sites need no new standing config. Empty disables notify-and-fetch fallback.
|
||||
/// </summary>
|
||||
public string CentralFetchBaseUrl { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// How long a staged PendingDeployment (and its fetch token) stays valid. Must
|
||||
/// comfortably cover both site nodes' fetches within one deploy window.
|
||||
/// </summary>
|
||||
public TimeSpan PendingDeploymentTtl { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>
|
||||
/// How often the central <c>PendingDeploymentPurgeActor</c> singleton reclaims
|
||||
/// expired (TTL-elapsed) PendingDeployment staging rows. Best-effort hygiene only:
|
||||
/// supersession bounds pending rows to ≤1 per instance and the config-fetch endpoint
|
||||
/// already enforces the TTL, so this purge merely sweeps rows left behind by instances
|
||||
/// that are deployed once and never re-deployed. Default 1 hour ≫ <see cref="PendingDeploymentTtl"/>.
|
||||
/// </summary>
|
||||
public TimeSpan PendingDeploymentPurgeInterval { get; set; } = TimeSpan.FromHours(1);
|
||||
}
|
||||
|
||||
@@ -116,19 +116,20 @@ public class CommunicationService
|
||||
// ── Pattern 1: Instance Deployment ──
|
||||
|
||||
/// <summary>
|
||||
/// Sends a deployment command for an instance to a site.
|
||||
/// Sends a small "refresh deployment" notify to a site (notify-and-fetch):
|
||||
/// the site fetches the config over HTTP rather than receiving it inline.
|
||||
/// Reply is the existing DeploymentStatusResponse, bounded by the deployment timeout.
|
||||
/// </summary>
|
||||
/// <param name="siteId">The target site identifier.</param>
|
||||
/// <param name="command">The deployment command.</param>
|
||||
/// <param name="command">The refresh-deployment notify.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The deployment status response.</returns>
|
||||
public async Task<DeploymentStatusResponse> DeployInstanceAsync(
|
||||
string siteId, DeployInstanceCommand command, CancellationToken cancellationToken = default)
|
||||
public async Task<DeploymentStatusResponse> RefreshDeploymentAsync(
|
||||
string siteId, RefreshDeploymentCommand command, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Sending DeployInstanceCommand to site {SiteId}, instance={Instance}, correlationId={DeploymentId}",
|
||||
_logger.LogInformation(
|
||||
"Sending RefreshDeploymentCommand to site {SiteId}, instance={Instance}, deploymentId={DeploymentId}",
|
||||
siteId, command.InstanceUniqueName, command.DeploymentId);
|
||||
|
||||
var envelope = new SiteEnvelope(siteId, command);
|
||||
return await GetActor().Ask<DeploymentStatusResponse>(
|
||||
envelope, _options.DeploymentTimeout, cancellationToken);
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// Central-side startup-reconciliation handler. A site node, on startup, reports its
|
||||
/// local deployed inventory via <see cref="ReconcileSiteRequest"/> (delivered over
|
||||
/// ClusterClient to <see cref="Actors.CentralCommunicationActor"/>); this service diffs
|
||||
/// it against central's expected deployed set and replies with fresh fetch tokens for the
|
||||
/// gap — instances the node is missing or has at a stale revision — plus the orphan names
|
||||
/// (present locally but no longer deployed centrally, which the node only logs).
|
||||
///
|
||||
/// This self-heals a node that was DOWN during a deploy and therefore missed the
|
||||
/// replicate. The node fetches each gap item's config over the existing token-gated HTTP
|
||||
/// endpoint; this service stages those tokens as <c>PendingDeployment</c> rows.
|
||||
///
|
||||
/// Scoped service (holds scoped repositories); resolved per-request by the actor inside a
|
||||
/// DI scope. The actor stays thin: it captures <c>Sender</c>, resolves this service in a
|
||||
/// scope, awaits <see cref="ReconcileAsync"/>, and pipes the response back.
|
||||
/// </summary>
|
||||
public class ReconcileService
|
||||
{
|
||||
private readonly IDeploymentManagerRepository _deploymentRepository;
|
||||
private readonly ISiteRepository _siteRepository;
|
||||
private readonly CommunicationOptions _options;
|
||||
private readonly ILogger<ReconcileService> _logger;
|
||||
|
||||
/// <summary>Initializes the reconciliation service.</summary>
|
||||
/// <param name="deploymentRepository">Repository for the expected-set query and pending-row staging.</param>
|
||||
/// <param name="siteRepository">Repository used to resolve a site's numeric id from its identifier.</param>
|
||||
/// <param name="options">Communication options carrying <see cref="CommunicationOptions.CentralFetchBaseUrl"/> and <see cref="CommunicationOptions.PendingDeploymentTtl"/>.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
public ReconcileService(
|
||||
IDeploymentManagerRepository deploymentRepository,
|
||||
ISiteRepository siteRepository,
|
||||
IOptions<CommunicationOptions> options,
|
||||
ILogger<ReconcileService> logger)
|
||||
{
|
||||
_deploymentRepository = deploymentRepository;
|
||||
_siteRepository = siteRepository;
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Diffs the node's reported inventory against central's expected deployed set,
|
||||
/// stages fresh fetch tokens for the gap, and returns the reconcile response.
|
||||
/// </summary>
|
||||
/// <param name="request">The site node's reconcile request (its local name→revision-hash map).</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>The gap (with fresh tokens), orphan names, and the central fetch base URL.</returns>
|
||||
public async Task<ReconcileSiteResponse> ReconcileAsync(
|
||||
ReconcileSiteRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var baseUrl = _options.CentralFetchBaseUrl;
|
||||
|
||||
// 1. Resolve the numeric site id. An unknown site is non-fatal: reply empty
|
||||
// (the node simply finds no gap to fetch) and log a warning so a
|
||||
// misconfigured SiteIdentifier is visible to operators.
|
||||
var site = await _siteRepository
|
||||
.GetSiteByIdentifierAsync(request.SiteIdentifier, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (site == null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Reconcile request from unknown site '{SiteIdentifier}' (node {NodeId}); replying with empty gap",
|
||||
request.SiteIdentifier, request.NodeId);
|
||||
return new ReconcileSiteResponse(
|
||||
Array.Empty<ReconcileGapItem>(), Array.Empty<string>(), baseUrl);
|
||||
}
|
||||
|
||||
// 2. Central's expected deployed set for this site (instances with a snapshot).
|
||||
var expected = await _deploymentRepository
|
||||
.GetExpectedDeploymentsForSiteAsync(site.Id, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var localMap = request.LocalNameToRevisionHash;
|
||||
var gap = new List<ReconcileGapItem>();
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var expiresAt = now + _options.PendingDeploymentTtl;
|
||||
|
||||
// 3. GAP = expected items the node is MISSING (name absent locally) or STALE
|
||||
// (local revision != expected revision). Current items are omitted.
|
||||
foreach (var exp in expected)
|
||||
{
|
||||
var present = localMap.TryGetValue(exp.InstanceUniqueName, out var localHash);
|
||||
var stale = present && !string.Equals(localHash, exp.RevisionHash, StringComparison.Ordinal);
|
||||
if (present && !stale)
|
||||
continue; // node already has the current revision
|
||||
|
||||
// 4. Read the frozen snapshot config to stage. Null = the snapshot was deleted
|
||||
// between the expected-set query and now (instance removed mid-reconcile);
|
||||
// skip it — best-effort reconcile re-runs on the next node startup.
|
||||
var snapshot = await _deploymentRepository
|
||||
.GetDeployedSnapshotByInstanceIdAsync(exp.InstanceId, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (snapshot == null)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Reconcile: snapshot for instance {Instance} disappeared (deleted race); skipping",
|
||||
exp.InstanceUniqueName);
|
||||
continue;
|
||||
}
|
||||
|
||||
var token = DeploymentFetchToken.Generate();
|
||||
|
||||
// Stage with the snapshot's DeploymentId as the deploymentId so the gap item's
|
||||
// DeploymentId + token point the node at the right pending row to fetch.
|
||||
//
|
||||
// StagePendingIfAbsent is insert-if-absent: if BOTH site nodes are concurrently
|
||||
// missing the same instance (e.g. fresh container start / cleared SQLite after a
|
||||
// successful deploy), both attempt to stage here. The first succeeds (true); the
|
||||
// second gets false and is handled in the !staged branch below — it returns the
|
||||
// existing pending row's token so it heals in the same round, rather than being
|
||||
// omitted. Deploy-time supersession serializes via the per-instance operation lock.
|
||||
var staged = await _deploymentRepository.StagePendingIfAbsentAsync(
|
||||
exp.InstanceId, snapshot.DeploymentId, exp.RevisionHash,
|
||||
snapshot.ConfigurationJson, token, now, expiresAt, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!staged)
|
||||
{
|
||||
// A pending row already exists for this instance — either a CONCURRENT reconcile
|
||||
// from the other site node (both nodes' SQLite empty after a fresh/cleared deploy,
|
||||
// both reconciling at startup) or an in-flight deploy. Do NOT omit the item: if we
|
||||
// did, the second concurrently-missing node would get 0 fetched and stay unhealed
|
||||
// until a later restart. Instead, read the EXISTING pending row and emit a gap item
|
||||
// carrying ITS DeploymentId/RevisionHash/Token. The fetch token is multi-use within
|
||||
// its TTL, so both nodes fetch the same pending config and heal in the same round.
|
||||
// (If the existing row is from an in-flight deploy its config is newer than the
|
||||
// snapshot — fetching it is still correct; the site's guarded write handles ordering.)
|
||||
var existing = await _deploymentRepository
|
||||
.GetPendingDeploymentByInstanceIdAsync(exp.InstanceId, now, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (existing != null)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Reconcile: pending row already exists for instance {Instance} (concurrent reconcile or in-flight deploy); returning existing token so this node heals too",
|
||||
exp.InstanceUniqueName);
|
||||
gap.Add(new ReconcileGapItem(
|
||||
exp.InstanceUniqueName, existing.DeploymentId, existing.RevisionHash,
|
||||
exp.IsEnabled, existing.Token));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Raced away: the pending row was purged between the stage attempt and this
|
||||
// read. Omit it — reconcile is best-effort and the node retries next round.
|
||||
_logger.LogDebug(
|
||||
"Reconcile: pending row for instance {Instance} disappeared between stage and read (purged race); omitting from gap",
|
||||
exp.InstanceUniqueName);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
gap.Add(new ReconcileGapItem(
|
||||
exp.InstanceUniqueName, snapshot.DeploymentId, exp.RevisionHash, exp.IsEnabled, token));
|
||||
}
|
||||
|
||||
// 5. ORPHANS = names the node has locally that central no longer considers deployed.
|
||||
// The node only LOGS these (never deletes).
|
||||
var expectedNames = new HashSet<string>(
|
||||
expected.Select(e => e.InstanceUniqueName), StringComparer.Ordinal);
|
||||
var orphans = localMap.Keys
|
||||
.Where(name => !expectedNames.Contains(name))
|
||||
.ToList();
|
||||
|
||||
_logger.LogDebug(
|
||||
"Reconcile for site {SiteIdentifier} (node {NodeId}): {GapCount} gap, {OrphanCount} orphan(s)",
|
||||
request.SiteIdentifier, request.NodeId, gap.Count, orphans.Count);
|
||||
|
||||
// 6. Reply.
|
||||
return new ReconcileSiteResponse(gap, orphans, baseUrl);
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,11 @@ public static class ServiceCollectionExtensions
|
||||
services.AddSingleton<SiteStreamGrpcClientFactory>();
|
||||
services.AddSingleton<DebugStreamService>();
|
||||
|
||||
// Startup reconciliation handler — scoped (holds scoped repositories), resolved
|
||||
// per-request by CentralCommunicationActor inside a DI scope. Harmless on site
|
||||
// hosts: only the central actor ever resolves it.
|
||||
services.AddScoped<ReconcileService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Configurations;
|
||||
|
||||
public class PendingDeploymentConfiguration : IEntityTypeConfiguration<PendingDeployment>
|
||||
{
|
||||
/// <summary>Configures the EF Core mapping for <see cref="PendingDeployment"/>.</summary>
|
||||
/// <param name="builder">The entity type builder.</param>
|
||||
public void Configure(EntityTypeBuilder<PendingDeployment> builder)
|
||||
{
|
||||
builder.ToTable("PendingDeployments");
|
||||
builder.HasKey(x => x.Id);
|
||||
builder.Property(x => x.DeploymentId).IsRequired().HasMaxLength(100);
|
||||
builder.HasIndex(x => x.DeploymentId).IsUnique();
|
||||
builder.HasIndex(x => x.InstanceId);
|
||||
builder.HasIndex(x => x.ExpiresAtUtc);
|
||||
builder.Property(x => x.RevisionHash).IsRequired().HasMaxLength(100);
|
||||
builder.Property(x => x.ConfigurationJson).IsRequired(); // nvarchar(max)
|
||||
builder.Property(x => x.Token).IsRequired().HasMaxLength(128);
|
||||
|
||||
builder.HasOne<Instance>().WithMany()
|
||||
.HasForeignKey(x => x.InstanceId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
}
|
||||
}
|
||||
+2009
File diff suppressed because it is too large
Load Diff
+63
@@ -0,0 +1,63 @@
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddPendingDeployment : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateTable(
|
||||
name: "PendingDeployments",
|
||||
columns: table => new
|
||||
{
|
||||
Id = table.Column<int>(type: "int", nullable: false)
|
||||
.Annotation("SqlServer:Identity", "1, 1"),
|
||||
DeploymentId = table.Column<string>(type: "nvarchar(100)", maxLength: 100, nullable: false),
|
||||
InstanceId = table.Column<int>(type: "int", nullable: false),
|
||||
RevisionHash = table.Column<string>(type: "nvarchar(100)", maxLength: 100, nullable: false),
|
||||
ConfigurationJson = table.Column<string>(type: "nvarchar(max)", nullable: false),
|
||||
Token = table.Column<string>(type: "nvarchar(128)", maxLength: 128, nullable: false),
|
||||
CreatedAtUtc = table.Column<DateTimeOffset>(type: "datetimeoffset", nullable: false),
|
||||
ExpiresAtUtc = table.Column<DateTimeOffset>(type: "datetimeoffset", nullable: false)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_PendingDeployments", x => x.Id);
|
||||
table.ForeignKey(
|
||||
name: "FK_PendingDeployments_Instances_InstanceId",
|
||||
column: x => x.InstanceId,
|
||||
principalTable: "Instances",
|
||||
principalColumn: "Id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_PendingDeployments_DeploymentId",
|
||||
table: "PendingDeployments",
|
||||
column: "DeploymentId",
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_PendingDeployments_ExpiresAtUtc",
|
||||
table: "PendingDeployments",
|
||||
column: "ExpiresAtUtc");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_PendingDeployments_InstanceId",
|
||||
table: "PendingDeployments",
|
||||
column: "InstanceId");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropTable(
|
||||
name: "PendingDeployments");
|
||||
}
|
||||
}
|
||||
}
|
||||
+57
@@ -267,6 +267,54 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
b.ToTable("DeploymentRecords");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment.PendingDeployment", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("int");
|
||||
|
||||
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<int>("Id"));
|
||||
|
||||
b.Property<string>("ConfigurationJson")
|
||||
.IsRequired()
|
||||
.HasColumnType("nvarchar(max)");
|
||||
|
||||
b.Property<DateTimeOffset>("CreatedAtUtc")
|
||||
.HasColumnType("datetimeoffset");
|
||||
|
||||
b.Property<string>("DeploymentId")
|
||||
.IsRequired()
|
||||
.HasMaxLength(100)
|
||||
.HasColumnType("nvarchar(100)");
|
||||
|
||||
b.Property<DateTimeOffset>("ExpiresAtUtc")
|
||||
.HasColumnType("datetimeoffset");
|
||||
|
||||
b.Property<int>("InstanceId")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<string>("RevisionHash")
|
||||
.IsRequired()
|
||||
.HasMaxLength(100)
|
||||
.HasColumnType("nvarchar(100)");
|
||||
|
||||
b.Property<string>("Token")
|
||||
.IsRequired()
|
||||
.HasMaxLength(128)
|
||||
.HasColumnType("nvarchar(128)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("DeploymentId")
|
||||
.IsUnique();
|
||||
|
||||
b.HasIndex("ExpiresAtUtc");
|
||||
|
||||
b.HasIndex("InstanceId");
|
||||
|
||||
b.ToTable("PendingDeployments", (string)null);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment.SystemArtifactDeploymentRecord", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
@@ -1720,6 +1768,15 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
.IsRequired();
|
||||
});
|
||||
|
||||
modelBuilder.Entity("ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment.PendingDeployment", b =>
|
||||
{
|
||||
b.HasOne("ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances.Instance", null)
|
||||
.WithMany()
|
||||
.HasForeignKey("InstanceId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
});
|
||||
|
||||
modelBuilder.Entity("ZB.MOM.WW.ScadaBridge.Commons.Entities.ExternalSystems.ExternalSystemMethod", b =>
|
||||
{
|
||||
b.HasOne("ZB.MOM.WW.ScadaBridge.Commons.Entities.ExternalSystems.ExternalSystemDefinition", null)
|
||||
|
||||
+143
@@ -2,6 +2,8 @@ using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
|
||||
@@ -197,6 +199,147 @@ public class DeploymentManagerRepository : IDeploymentManagerRepository
|
||||
}
|
||||
}
|
||||
|
||||
// --- Notify-and-fetch: PendingDeployment staging store ---
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task AddPendingDeploymentAsync(PendingDeployment pending, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(pending);
|
||||
|
||||
// Supersession: a newer deploy for the same instance replaces any prior pending
|
||||
// row. Safe because the per-instance operation lock serializes same-instance
|
||||
// deploys. SaveChanges is deferred to the caller, matching the other Add methods.
|
||||
var prior = await _dbContext.Set<PendingDeployment>()
|
||||
.Where(p => p.InstanceId == pending.InstanceId)
|
||||
.ToListAsync(cancellationToken);
|
||||
if (prior.Count > 0)
|
||||
{
|
||||
_dbContext.Set<PendingDeployment>().RemoveRange(prior);
|
||||
}
|
||||
await _dbContext.Set<PendingDeployment>().AddAsync(pending, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<PendingDeployment?> GetPendingDeploymentByIdAsync(string deploymentId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return _dbContext.Set<PendingDeployment>()
|
||||
.FirstOrDefaultAsync(p => p.DeploymentId == deploymentId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<PendingDeployment?> GetPendingDeploymentByInstanceIdAsync(int instanceId, DateTimeOffset? nowUtc = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// At most one pending row per instance by design (supersession + stage-if-absent),
|
||||
// but order deterministically and take the most recent so a hypothetical duplicate
|
||||
// never makes the read non-deterministic — mirrors GetCurrentDeploymentStatusAsync.
|
||||
var query = _dbContext.Set<PendingDeployment>()
|
||||
.Where(p => p.InstanceId == instanceId);
|
||||
|
||||
// Expiry-aware: never hand back an EXPIRED row. Pending rows are only TTL-purged, so an
|
||||
// expired-but-unpurged row would otherwise return a token the config-fetch endpoint 404s
|
||||
// (it correctly rejects expired rows), leaving the node unhealed.
|
||||
if (nowUtc.HasValue)
|
||||
{
|
||||
var now = nowUtc.Value;
|
||||
query = query.Where(p => p.ExpiresAtUtc > now);
|
||||
}
|
||||
|
||||
return query
|
||||
.OrderByDescending(p => p.CreatedAtUtc)
|
||||
.ThenByDescending(p => p.Id)
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task DeletePendingDeploymentByIdAsync(string deploymentId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var row = await _dbContext.Set<PendingDeployment>()
|
||||
.FirstOrDefaultAsync(p => p.DeploymentId == deploymentId, cancellationToken);
|
||||
if (row != null)
|
||||
{
|
||||
_dbContext.Set<PendingDeployment>().Remove(row);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<int> PurgeExpiredPendingDeploymentsAsync(DateTimeOffset nowUtc, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Self-contained TTL maintenance op: commits its own delete and returns the count.
|
||||
var expired = await _dbContext.Set<PendingDeployment>()
|
||||
.Where(p => p.ExpiresAtUtc <= nowUtc)
|
||||
.ToListAsync(cancellationToken);
|
||||
if (expired.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
_dbContext.Set<PendingDeployment>().RemoveRange(expired);
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
return expired.Count;
|
||||
}
|
||||
|
||||
// --- Startup reconciliation ---
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<ExpectedDeployment>> GetExpectedDeploymentsForSiteAsync(int siteId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Inner join: only instances that have a DeployedConfigSnapshot are returned.
|
||||
// ConfigurationJson is intentionally excluded — the caller fetches the full config
|
||||
// on demand via the HTTP endpoint for gap instances only.
|
||||
return await (
|
||||
from snap in _dbContext.Set<DeployedConfigSnapshot>()
|
||||
join inst in _dbContext.Set<Instance>() on snap.InstanceId equals inst.Id
|
||||
where inst.SiteId == siteId
|
||||
select new ExpectedDeployment(
|
||||
inst.Id,
|
||||
inst.UniqueName,
|
||||
snap.RevisionHash,
|
||||
snap.DeploymentId,
|
||||
inst.State == InstanceState.Enabled))
|
||||
.ToListAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> StagePendingIfAbsentAsync(
|
||||
int instanceId, string deploymentId, string revisionHash,
|
||||
string configurationJson, string token,
|
||||
DateTimeOffset createdAtUtc, DateTimeOffset expiresAtUtc,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Treat createdAtUtc as "now". FIRST remove any EXPIRED pending rows for this instance
|
||||
// (ExpiresAtUtc <= now). Pending rows are only TTL-purged (the periodic purge is still a
|
||||
// deferred TODO), so an EXPIRED-but-unpurged row would otherwise (a) read as "a deploy is
|
||||
// in flight" and block staging — handing the node an expired token (HTTP 404) and leaving
|
||||
// it unhealed — and (b) collide on the DeploymentId UNIQUE index when a reconcile re-stages
|
||||
// the snapshot's own DeploymentId. Dropping expired rows first fixes both.
|
||||
var expired = await _dbContext.Set<PendingDeployment>()
|
||||
.Where(p => p.InstanceId == instanceId && p.ExpiresAtUtc <= createdAtUtc)
|
||||
.ToListAsync(cancellationToken);
|
||||
if (expired.Count > 0)
|
||||
{
|
||||
_dbContext.Set<PendingDeployment>().RemoveRange(expired);
|
||||
}
|
||||
|
||||
// THEN insert-if-absent against still-LIVE rows only. A live pending row means a genuine
|
||||
// in-flight deploy (or a concurrent reconcile) already owns the slot — do NOT supersede it;
|
||||
// clobbering it could make the node fetch the reconcile token while the original deliver is
|
||||
// mid-flight. (Expired rows just removed are disjoint from this future-expiry predicate.)
|
||||
var liveExists = await _dbContext.Set<PendingDeployment>()
|
||||
.AnyAsync(p => p.InstanceId == instanceId && p.ExpiresAtUtc > createdAtUtc, cancellationToken);
|
||||
if (!liveExists)
|
||||
{
|
||||
var pending = new PendingDeployment(
|
||||
deploymentId, instanceId, revisionHash,
|
||||
configurationJson, token, createdAtUtc, expiresAtUtc);
|
||||
await _dbContext.Set<PendingDeployment>().AddAsync(pending, cancellationToken);
|
||||
}
|
||||
|
||||
// Self-contained: one SaveChanges flushes the expired-row cleanup and, when staged, the new
|
||||
// row together (EF orders the delete before the same-DeploymentId insert to satisfy the
|
||||
// unique index). Returns true only when a fresh row was staged.
|
||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||
return !liveExists;
|
||||
}
|
||||
|
||||
// --- Instance lookups for deployment pipeline ---
|
||||
|
||||
/// <inheritdoc />
|
||||
|
||||
@@ -89,6 +89,8 @@ public class ScadaBridgeDbContext : DbContext, IDataProtectionKeyContext
|
||||
public DbSet<SystemArtifactDeploymentRecord> SystemArtifactDeploymentRecords => Set<SystemArtifactDeploymentRecord>();
|
||||
/// <summary>Gets the set of deployed configuration snapshots.</summary>
|
||||
public DbSet<DeployedConfigSnapshot> DeployedConfigSnapshots => Set<DeployedConfigSnapshot>();
|
||||
/// <summary>Gets the set of pending deployments staged for HTTP fetch by sites.</summary>
|
||||
public DbSet<PendingDeployment> PendingDeployments => Set<PendingDeployment>();
|
||||
|
||||
// External Systems
|
||||
/// <summary>Gets the set of external system definitions.</summary>
|
||||
|
||||
@@ -9,6 +9,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
@@ -25,7 +26,7 @@ namespace ZB.MOM.WW.ScadaBridge.DeploymentManager;
|
||||
/// 3. Flatten configuration via TemplateEngine (captures template state at time of flatten -- WP-16)
|
||||
/// 4. Validate flattened configuration
|
||||
/// 5. Compute revision hash and diff
|
||||
/// 6. Send DeployInstanceCommand to site via CommunicationService
|
||||
/// 6. Stage a PendingDeployment row + send RefreshDeploymentCommand (notify-and-fetch; site fetches the config over HTTP)
|
||||
/// 7. Track deployment status with optimistic concurrency (WP-4)
|
||||
/// 8. Store deployed config snapshot (WP-8)
|
||||
/// 9. Audit log all actions
|
||||
@@ -45,6 +46,7 @@ public class DeploymentService
|
||||
private readonly RevisionHashService _revisionHashService;
|
||||
private readonly IDeploymentStatusNotifier _statusNotifier;
|
||||
private readonly DeploymentManagerOptions _options;
|
||||
private readonly CommunicationOptions _commOptions;
|
||||
private readonly ILogger<DeploymentService> _logger;
|
||||
|
||||
/// <summary>
|
||||
@@ -72,6 +74,13 @@ public class DeploymentService
|
||||
/// </param>
|
||||
/// <param name="statusNotifier">Notifier for pushing deployment status changes to the UI.</param>
|
||||
/// <param name="options">Deployment manager configuration options.</param>
|
||||
/// <param name="communicationOptions">
|
||||
/// Central-site communication options. The notify-and-fetch deploy path reads
|
||||
/// <see cref="CommunicationOptions.CentralFetchBaseUrl"/> (carried in the
|
||||
/// <c>RefreshDeploymentCommand</c> so sites need no standing config) and
|
||||
/// <see cref="CommunicationOptions.PendingDeploymentTtl"/> (the staged
|
||||
/// <c>PendingDeployment</c> row's lifetime).
|
||||
/// </param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
public DeploymentService(
|
||||
IDeploymentManagerRepository repository,
|
||||
@@ -84,6 +93,7 @@ public class DeploymentService
|
||||
RevisionHashService revisionHashService,
|
||||
IDeploymentStatusNotifier statusNotifier,
|
||||
IOptions<DeploymentManagerOptions> options,
|
||||
IOptions<CommunicationOptions> communicationOptions,
|
||||
ILogger<DeploymentService> logger)
|
||||
{
|
||||
_repository = repository;
|
||||
@@ -96,6 +106,7 @@ public class DeploymentService
|
||||
_revisionHashService = revisionHashService;
|
||||
_statusNotifier = statusNotifier;
|
||||
_options = options.Value;
|
||||
_commOptions = communicationOptions.Value;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -209,6 +220,16 @@ public class DeploymentService
|
||||
if (reconciled != null)
|
||||
return Result<DeploymentRecord>.Success(reconciled);
|
||||
|
||||
// Notify-and-fetch: the site fetches the staged config from
|
||||
// CentralFetchBaseUrl, so a deploy is impossible without it. Fail fast
|
||||
// here — BEFORE creating an InProgress record or staging a pending row —
|
||||
// so the operator sees a clear configuration error instead of a
|
||||
// confusing downstream site-fetch failure (and no InProgress record is
|
||||
// stranded).
|
||||
if (string.IsNullOrEmpty(_commOptions.CentralFetchBaseUrl))
|
||||
return Result<DeploymentRecord>.Failure(
|
||||
"CentralFetchBaseUrl is not configured — required for deployment (notify-and-fetch).");
|
||||
|
||||
// WP-4: Create the deployment record directly in InProgress.
|
||||
//
|
||||
// DeploymentManager-022: the previous code wrote the record as Pending,
|
||||
@@ -236,16 +257,38 @@ public class DeploymentService
|
||||
|
||||
try
|
||||
{
|
||||
// WP-1: Send to site via CommunicationService
|
||||
// Notify-and-fetch: instead of shipping the (potentially oversized,
|
||||
// silently-dropped >128 KB) flattened config inline in a
|
||||
// DeployInstanceCommand, stage it in a PendingDeployment row and send
|
||||
// a small RefreshDeploymentCommand. The site fetches the config from
|
||||
// CentralFetchBaseUrl over HTTP using the per-deployment fetch token.
|
||||
var siteId = await ResolveSiteIdentifierAsync(instance.SiteId, cancellationToken);
|
||||
var command = new DeployInstanceCommand(
|
||||
deploymentId, instance.UniqueName, revisionHash, configJson, user, DateTimeOffset.UtcNow);
|
||||
|
||||
var token = DeploymentFetchToken.Generate();
|
||||
var stagedAt = DateTimeOffset.UtcNow;
|
||||
await _repository.AddPendingDeploymentAsync(new PendingDeployment(
|
||||
deploymentId, instanceId, revisionHash, configJson, token,
|
||||
stagedAt, stagedAt + _commOptions.PendingDeploymentTtl), cancellationToken);
|
||||
await _repository.SaveChangesAsync(cancellationToken);
|
||||
|
||||
var command = new RefreshDeploymentCommand(
|
||||
deploymentId, instance.UniqueName, revisionHash, user, stagedAt,
|
||||
_commOptions.CentralFetchBaseUrl, token);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Sending deployment {DeploymentId} for instance {Instance} to site {SiteId}",
|
||||
"Sending deployment {DeploymentId} for instance {Instance} to site {SiteId} (notify-and-fetch)",
|
||||
deploymentId, instance.UniqueName, siteId);
|
||||
|
||||
var response = await _communicationService.DeployInstanceAsync(siteId, command, cancellationToken);
|
||||
// Cleanup of the staged PendingDeployment is TTL-based ONLY — the row
|
||||
// is deliberately NOT deleted on success or in the catch. On a
|
||||
// central-side Ask timeout the site may have applied AND told the
|
||||
// standby node to fetch; deleting now would 404 that in-flight
|
||||
// standby fetch and break failover. Supersession bounds pending rows
|
||||
// to ≤1 per instance and the fetch endpoint enforces the TTL, so
|
||||
// leaving rows for TTL purge is safe. Expired rows are swept by the
|
||||
// central PendingDeploymentPurgeActor singleton on its
|
||||
// CommunicationOptions.PendingDeploymentPurgeInterval cadence.
|
||||
var response = await _communicationService.RefreshDeploymentAsync(siteId, command, cancellationToken);
|
||||
|
||||
// WP-1: Update status based on site response.
|
||||
record.Status = response.Status;
|
||||
@@ -309,7 +352,7 @@ public class DeploymentService
|
||||
// timed out, that token is already cancelled and the cleanup writes
|
||||
// would themselves throw before the Failed status is persisted.
|
||||
// Use CancellationToken.None so the failure is durably recorded.
|
||||
var isTimeout = ex is TimeoutException or OperationCanceledException;
|
||||
var isTimeout = ex is TimeoutException or OperationCanceledException or Akka.Actor.AskTimeoutException;
|
||||
|
||||
record.Status = DeploymentStatus.Failed;
|
||||
record.ErrorMessage = isTimeout
|
||||
@@ -320,6 +363,12 @@ public class DeploymentService
|
||||
try
|
||||
{
|
||||
await _repository.UpdateDeploymentRecordAsync(record, CancellationToken.None);
|
||||
// Note: if the staging SaveChangesAsync above was interrupted, an
|
||||
// Added PendingDeployment may still be tracked and will be
|
||||
// committed by this cleanup save. That row is orphaned (no
|
||||
// RefreshDeploymentCommand was sent, so no site holds its token)
|
||||
// and is removed by TTL purge / superseded by the next deploy --
|
||||
// harmless.
|
||||
await _repository.SaveChangesAsync(CancellationToken.None);
|
||||
NotifyStatusChange(record);
|
||||
|
||||
|
||||
@@ -756,6 +756,60 @@ akka {{
|
||||
_actorSystem.ActorOf(kpiHistoryProxyProps, "kpi-history-recorder-proxy");
|
||||
_logger.LogInformation("KpiHistoryRecorderActor singleton created (not readiness-gated)");
|
||||
|
||||
// Notify-and-fetch deploy transport — central singleton that periodically
|
||||
// reclaims expired (TTL-elapsed) PendingDeployment staging rows. Mirrors the
|
||||
// kpi-history-recorder singleton pattern above: a ClusterSingletonManager pins
|
||||
// it to the active central node, a ClusterSingletonProxy gives a stable address,
|
||||
// and a PhaseClusterLeave graceful-stop task drains the in-flight tick before
|
||||
// handover. The purge timer self-schedules in PreStart. NOT readiness-gated by
|
||||
// design: the purge is best-effort hygiene — supersession bounds the table to ≤1
|
||||
// pending row per instance and the config-fetch endpoint enforces the TTL on read,
|
||||
// so correctness never depends on it; pending-deployment-purge is therefore
|
||||
// deliberately absent from RequiredSingletonsHealthCheck. The actor takes the root
|
||||
// IServiceProvider and opens its own per-tick DI scope (IDeploymentManagerRepository
|
||||
// is a scoped EF Core service).
|
||||
var pendingPurgeLogger = _serviceProvider.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger<PendingDeploymentPurgeActor>();
|
||||
var pendingPurgeCommunicationOptions =
|
||||
_serviceProvider.GetRequiredService<IOptions<CommunicationOptions>>();
|
||||
|
||||
var pendingPurgeSingletonProps = ClusterSingletonManager.Props(
|
||||
singletonProps: Props.Create(() => new PendingDeploymentPurgeActor(
|
||||
_serviceProvider,
|
||||
pendingPurgeCommunicationOptions,
|
||||
pendingPurgeLogger)),
|
||||
terminationMessage: PoisonPill.Instance,
|
||||
settings: ClusterSingletonManagerSettings.Create(_actorSystem!)
|
||||
.WithSingletonName("pending-deployment-purge"));
|
||||
var pendingPurgeSingletonManager =
|
||||
_actorSystem!.ActorOf(pendingPurgeSingletonProps, "pending-deployment-purge-singleton");
|
||||
|
||||
var pendingPurgeShutdown = Akka.Actor.CoordinatedShutdown.Get(_actorSystem);
|
||||
pendingPurgeShutdown.AddTask(
|
||||
Akka.Actor.CoordinatedShutdown.PhaseClusterLeave,
|
||||
"drain-pending-deployment-purge-singleton",
|
||||
async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await pendingPurgeSingletonManager.GracefulStop(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"PendingDeploymentPurge singleton did not drain within the graceful-stop "
|
||||
+ "timeout; falling through to PoisonPill handover");
|
||||
}
|
||||
return Akka.Done.Instance;
|
||||
});
|
||||
|
||||
var pendingPurgeProxyProps = ClusterSingletonProxy.Props(
|
||||
singletonManagerPath: "/user/pending-deployment-purge-singleton",
|
||||
settings: ClusterSingletonProxySettings.Create(_actorSystem)
|
||||
.WithSingletonName("pending-deployment-purge"));
|
||||
_actorSystem.ActorOf(pendingPurgeProxyProps, "pending-deployment-purge-proxy");
|
||||
_logger.LogInformation("PendingDeploymentPurgeActor singleton created (not readiness-gated)");
|
||||
|
||||
_logger.LogInformation("Central actors registered. CentralCommunicationActor created.");
|
||||
}
|
||||
|
||||
@@ -819,6 +873,14 @@ akka {{
|
||||
var siteHealthCollector = _serviceProvider.GetService<ZB.MOM.WW.ScadaBridge.HealthMonitoring.ISiteHealthCollector>();
|
||||
siteHealthCollector?.SetNodeHostname(_nodeOptions.NodeHostname);
|
||||
|
||||
// Notify-and-fetch: the deployment config fetcher pulls a deployment's flattened
|
||||
// config from central over HTTP. Used by BOTH the active singleton
|
||||
// (RefreshDeploymentCommand, Task 10) AND the standby replication path — the active
|
||||
// node now replicates only the deployment id and the standby fetches the config
|
||||
// itself, so a large config never crosses the intra-site Akka hop. Resolve once.
|
||||
var deploymentConfigFetcher =
|
||||
_serviceProvider.GetService<ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment.IDeploymentConfigFetcher>();
|
||||
|
||||
// Create SiteReplicationActor on every node (not a singleton)
|
||||
var sfStorage = _serviceProvider.GetRequiredService<StoreAndForwardStorage>();
|
||||
var replicationService = _serviceProvider.GetRequiredService<ReplicationService>();
|
||||
@@ -827,7 +889,8 @@ akka {{
|
||||
|
||||
var replicationActor = _actorSystem!.ActorOf(
|
||||
Props.Create(() => new SiteReplicationActor(
|
||||
storage, sfStorage, replicationService, siteRole, replicationLogger)),
|
||||
storage, sfStorage, replicationService, siteRole, replicationLogger,
|
||||
deploymentConfigFetcher)),
|
||||
"site-replication");
|
||||
|
||||
// Wire S&F replication handler to forward operations via the replication actor
|
||||
@@ -851,7 +914,9 @@ akka {{
|
||||
dclManager,
|
||||
replicationActor,
|
||||
siteHealthCollector,
|
||||
_serviceProvider)),
|
||||
_serviceProvider,
|
||||
null,
|
||||
deploymentConfigFetcher)),
|
||||
terminationMessage: PoisonPill.Instance,
|
||||
settings: ClusterSingletonManagerSettings.Create(_actorSystem!)
|
||||
.WithRole(siteRole)
|
||||
@@ -1005,6 +1070,39 @@ akka {{
|
||||
contacts.Count, _nodeOptions.SiteId);
|
||||
}
|
||||
|
||||
// Task 18c — per-node startup reconciliation. Created on EVERY site node (NOT a
|
||||
// singleton) so a standby that was DOWN during a deploy self-heals on its next
|
||||
// restart: it reports its local deployed inventory to central via the
|
||||
// SiteCommunicationActor Ask, fetches the gap (missing/stale) over HTTP, and
|
||||
// guarded-writes it (orphans are logged, never deleted). Requires the HTTP
|
||||
// config fetcher; if it is somehow absent the self-heal is skipped (best-effort —
|
||||
// replication remains the primary path and the next restart retries).
|
||||
if (deploymentConfigFetcher != null)
|
||||
{
|
||||
var reconcileLogger = _serviceProvider.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger<SiteReconciliationActor>();
|
||||
_actorSystem.ActorOf(
|
||||
Props.Create(() => new SiteReconciliationActor(
|
||||
storage,
|
||||
deploymentConfigFetcher,
|
||||
siteCommActor,
|
||||
_nodeOptions.SiteId!,
|
||||
_nodeOptions.NodeName,
|
||||
reconcileLogger,
|
||||
null,
|
||||
null)),
|
||||
"site-reconciliation");
|
||||
_logger.LogInformation(
|
||||
"SiteReconciliationActor created (per-node startup self-heal) for site {SiteId} node {Node}",
|
||||
_nodeOptions.SiteId, _nodeOptions.NodeName);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"No IDeploymentConfigFetcher available; SiteReconciliationActor not created — "
|
||||
+ "startup self-heal disabled (replication remains the primary path)");
|
||||
}
|
||||
|
||||
// Audit Log (#23) — site-side telemetry actor that drains the SQLite
|
||||
// Pending queue and pushes to central via IngestAuditEvents. Not a
|
||||
// cluster singleton: each site is its own cluster, and the actor reads
|
||||
|
||||
@@ -369,6 +369,10 @@ try
|
||||
// Basic-Auth + LDAP mechanism as /management; gated on the OperationalAudit
|
||||
// / AuditExport role sets.
|
||||
app.MapAuditAPI();
|
||||
// Notify-and-fetch deploy (#2/#3): site-facing token-gated fetch of a staged
|
||||
// deployment's flattened config. Machine-to-machine — AllowAnonymous, gated
|
||||
// solely by the per-deployment X-Deployment-Token (no central FallbackPolicy).
|
||||
app.MapDeploymentConfigAPI();
|
||||
app.MapHub<ZB.MOM.WW.ScadaBridge.ManagementService.DebugStreamHub>("/hubs/debug-stream");
|
||||
|
||||
// Compile and register all Inbound API method scripts at startup
|
||||
|
||||
@@ -41,7 +41,8 @@
|
||||
"LifecycleTimeout": "00:00:30",
|
||||
"QueryTimeout": "00:00:30",
|
||||
"TransportHeartbeatInterval": "00:00:05",
|
||||
"TransportFailureThreshold": "00:00:15"
|
||||
"TransportFailureThreshold": "00:00:15",
|
||||
"CentralFetchBaseUrl": "http://localhost:5000"
|
||||
},
|
||||
"HealthMonitoring": {
|
||||
"ReportInterval": "00:00:30",
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ManagementService;
|
||||
|
||||
/// <summary>
|
||||
/// Minimal-API endpoint a SITE calls to fetch a single deployment's flattened
|
||||
/// configuration by id (the "notify-and-fetch" deploy path): central stages a
|
||||
/// <see cref="PendingDeployment"/> row, then notifies the site of the deployment
|
||||
/// id; the site fetches the staged JSON over HTTP rather than receiving it inside
|
||||
/// an Akka message (avoids the 128 KB frame limit).
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Route.</b> <c>GET /api/internal/deployments/{deploymentId}/config</c>, mapped
|
||||
/// in the same central-role block as <c>/api/audit/*</c> and <c>/management</c>, so
|
||||
/// it is reachable on the same host/port (and through Traefik to the active node).
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Auth (security boundary).</b> This is a machine-to-machine call — NOT a
|
||||
/// cookie/JWT/LDAP-authenticated operator request. It is gated SOLELY by a
|
||||
/// per-deployment fetch token the site presents in the <c>X-Deployment-Token</c>
|
||||
/// header, compared against the staged row's token in constant time
|
||||
/// (<see cref="DeploymentFetchToken.ConstantTimeEquals"/>). The endpoint is marked
|
||||
/// <c>.AllowAnonymous()</c>; the central host registers NO authorization
|
||||
/// <c>FallbackPolicy</c>, so an anonymous endpoint is reachable without the
|
||||
/// per-operator Basic/LDAP flow the audit/management endpoints apply by hand.
|
||||
/// The token — short-TTL, single-deployment-scoped, removed on supersession — is
|
||||
/// the entire security boundary.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Existence hiding.</b> Unknown, superseded (the row is deleted), and expired
|
||||
/// deployments are indistinguishable (all <c>404 NotFound</c>) to any caller,
|
||||
/// regardless of the presented token. A live, non-expired row with a missing/wrong
|
||||
/// token returns <c>401</c> — which DOES confirm the id exists — and that is
|
||||
/// acceptable because deployment ids are unguessable GUIDs and the 256-bit
|
||||
/// per-deployment token is the real security boundary.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class DeploymentConfigEndpoints
|
||||
{
|
||||
/// <summary>Request header carrying the per-deployment fetch token.</summary>
|
||||
public const string TokenHeader = "X-Deployment-Token";
|
||||
|
||||
/// <summary>
|
||||
/// Registers the <c>GET /api/internal/deployments/{deploymentId}/config</c>
|
||||
/// token-gated fetch endpoint. Marked <c>AllowAnonymous</c> — the per-deployment
|
||||
/// token is the sole security boundary (see the type-level remarks).
|
||||
/// </summary>
|
||||
/// <param name="endpoints">The endpoint route builder to register the route on.</param>
|
||||
/// <returns>The same <paramref name="endpoints"/> builder, for chaining.</returns>
|
||||
public static IEndpointRouteBuilder MapDeploymentConfigAPI(this IEndpointRouteBuilder endpoints)
|
||||
{
|
||||
endpoints.MapGet("/api/internal/deployments/{deploymentId}/config", (Delegate)HandleGetConfig)
|
||||
.AllowAnonymous();
|
||||
return endpoints;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thin HTTP adapter: pulls the <c>X-Deployment-Token</c> header and the current
|
||||
/// UTC instant, loads the staged row, and delegates the security decision to the
|
||||
/// pure <see cref="Resolve"/> resolver.
|
||||
/// </summary>
|
||||
/// <param name="deploymentId">The deployment id route value (the fetch key).</param>
|
||||
/// <param name="ctx">The HTTP context for the current request.</param>
|
||||
/// <param name="repo">The deployment-manager repository resolving the staged row.</param>
|
||||
/// <param name="ct">A cancellation token tied to the request lifetime.</param>
|
||||
/// <returns>The HTTP result (200 staged JSON, 401, or 404).</returns>
|
||||
internal static async Task<IResult> HandleGetConfig(
|
||||
string deploymentId,
|
||||
HttpContext ctx,
|
||||
IDeploymentManagerRepository repo,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var token = ctx.Request.Headers[TokenHeader].ToString();
|
||||
var row = await repo.GetPendingDeploymentByIdAsync(deploymentId, ct);
|
||||
return Resolve(row, token, DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure, security-critical decision resolver for the fetch endpoint. Order of
|
||||
/// checks matters: existence/TTL are evaluated BEFORE the token so a missing,
|
||||
/// superseded, or expired deployment is indistinguishable (all <c>404</c>) to a
|
||||
/// caller without a valid token.
|
||||
/// </summary>
|
||||
/// <param name="row">The staged pending deployment, or <c>null</c> if unknown/superseded.</param>
|
||||
/// <param name="token">The token presented in the <c>X-Deployment-Token</c> header (may be empty).</param>
|
||||
/// <param name="nowUtc">The current UTC instant used for the TTL comparison.</param>
|
||||
/// <returns>
|
||||
/// <c>404 NotFound</c> when <paramref name="row"/> is null or expired;
|
||||
/// <c>401 Unauthorized</c> when the token is missing/empty or does not match;
|
||||
/// otherwise <c>200</c> with the staged <see cref="PendingDeployment.ConfigurationJson"/>
|
||||
/// as <c>application/json</c>.
|
||||
/// </returns>
|
||||
public static IResult Resolve(PendingDeployment? row, string token, DateTimeOffset nowUtc)
|
||||
{
|
||||
// Unknown OR superseded (supersession deletes the row): hide existence.
|
||||
if (row is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
// Expired (TTL elapsed): treat as gone — also hides existence to a wrong token.
|
||||
if (row.ExpiresAtUtc <= nowUtc)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
// Token is the entire security boundary; compare in constant time.
|
||||
if (string.IsNullOrEmpty(token) || !DeploymentFetchToken.ConstantTimeEquals(token, row.Token))
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
return Results.Content(row.ConfigurationJson, "application/json");
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
@@ -50,6 +51,12 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
private readonly IActorRef? _replicationActor;
|
||||
private readonly ISiteHealthCollector? _healthCollector;
|
||||
private readonly IServiceProvider? _serviceProvider;
|
||||
/// <summary>
|
||||
/// Notify-and-fetch (Task 10): fetches a deployment's flattened config from central
|
||||
/// over HTTP when a <see cref="RefreshDeploymentCommand"/> arrives. Optional — null on
|
||||
/// nodes/tests that never receive a refresh; the active site path supplies it via DI.
|
||||
/// </summary>
|
||||
private readonly IDeploymentConfigFetcher? _configFetcher;
|
||||
private readonly Dictionary<string, IActorRef> _instanceActors = new();
|
||||
/// <summary>
|
||||
/// Tracks Instance Actors that are terminating as part of a redeployment, keyed by
|
||||
@@ -97,6 +104,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// <param name="healthCollector">Optional site health collector.</param>
|
||||
/// <param name="serviceProvider">Optional service provider for resolving per-instance services.</param>
|
||||
/// <param name="loggerFactory">Optional logger factory for creating Instance Actor loggers.</param>
|
||||
/// <param name="configFetcher">
|
||||
/// Optional notify-and-fetch config fetcher (Task 10). Required for the
|
||||
/// <see cref="RefreshDeploymentCommand"/> path; null on nodes/tests that never
|
||||
/// receive a refresh.
|
||||
/// </param>
|
||||
public DeploymentManagerActor(
|
||||
SiteStorageService storage,
|
||||
ScriptCompilationService compilationService,
|
||||
@@ -108,7 +120,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
IActorRef? replicationActor = null,
|
||||
ISiteHealthCollector? healthCollector = null,
|
||||
IServiceProvider? serviceProvider = null,
|
||||
ILoggerFactory? loggerFactory = null)
|
||||
ILoggerFactory? loggerFactory = null,
|
||||
IDeploymentConfigFetcher? configFetcher = null)
|
||||
{
|
||||
_storage = storage;
|
||||
_compilationService = compilationService;
|
||||
@@ -119,6 +132,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
_replicationActor = replicationActor;
|
||||
_healthCollector = healthCollector;
|
||||
_serviceProvider = serviceProvider;
|
||||
_configFetcher = configFetcher;
|
||||
_logger = logger;
|
||||
// SiteRuntime-015: reuse a single logger factory for all Instance Actors.
|
||||
// Prefer an explicitly injected factory, fall back to one resolved from
|
||||
@@ -129,11 +143,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
?? Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance;
|
||||
|
||||
// Lifecycle commands
|
||||
Receive<DeployInstanceCommand>(HandleDeploy);
|
||||
Receive<DeployInstanceCommand>(cmd => HandleDeploy(cmd, Sender));
|
||||
Receive<DisableInstanceCommand>(HandleDisable);
|
||||
Receive<EnableInstanceCommand>(HandleEnable);
|
||||
Receive<DeleteInstanceCommand>(HandleDelete);
|
||||
|
||||
// Notify-and-fetch (Task 10): central sends a small RefreshDeploymentCommand;
|
||||
// the active singleton fetches the flattened config over HTTP, then reuses the
|
||||
// existing apply path. The two internal results carry the fetched config (or the
|
||||
// fetch error) back onto the actor thread along with the captured original sender.
|
||||
Receive<RefreshDeploymentCommand>(HandleRefreshDeployment);
|
||||
Receive<RefreshFetched>(HandleRefreshFetched);
|
||||
Receive<RefreshFetchFailed>(HandleRefreshFetchFailed);
|
||||
|
||||
// DeploymentManager-006: query-the-site-before-redeploy idempotency.
|
||||
// Central asks for the instance's currently-applied deployment identity
|
||||
// before re-sending a deployment whose prior record is stuck InProgress
|
||||
@@ -356,7 +378,15 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// terminating child is watched and the in-flight command is buffered until the
|
||||
/// <see cref="Terminated"/> signal arrives.
|
||||
/// </summary>
|
||||
private void HandleDeploy(DeployInstanceCommand command)
|
||||
/// <param name="replyTo">
|
||||
/// The actor to reply to with the eventual <see cref="DeploymentStatusResponse"/>.
|
||||
/// Passed explicitly (rather than read from <see cref="ActorBase.Sender"/>) so the
|
||||
/// notify-and-fetch path (<see cref="HandleRefreshFetched"/>) can supply the ORIGINAL
|
||||
/// central sender after the async config fetch, where <c>Sender</c> is no longer valid.
|
||||
/// The redeploy-buffer path carries it on <see cref="PendingRedeploy"/> so the buffered
|
||||
/// apply still replies to the right actor.
|
||||
/// </param>
|
||||
private void HandleDeploy(DeployInstanceCommand command, IActorRef replyTo)
|
||||
{
|
||||
var instanceName = command.InstanceUniqueName;
|
||||
_logger.LogInformation(
|
||||
@@ -369,7 +399,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
if (_instanceActors.TryGetValue(instanceName, out var existing))
|
||||
{
|
||||
_instanceActors.Remove(instanceName);
|
||||
_pendingRedeploys[existing] = new PendingRedeploy(command, Sender);
|
||||
_pendingRedeploys[existing] = new PendingRedeploy(command, replyTo);
|
||||
_terminatingActorsByName[instanceName] = existing;
|
||||
Context.Watch(existing);
|
||||
Context.Stop(existing);
|
||||
@@ -396,12 +426,12 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
$"superseded by newer deployment {command.DeploymentId} before predecessor finished terminating",
|
||||
DateTimeOffset.UtcNow));
|
||||
}
|
||||
_pendingRedeploys[terminatingRef] = new PendingRedeploy(command, Sender);
|
||||
_pendingRedeploys[terminatingRef] = new PendingRedeploy(command, replyTo);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fresh deployment — no existing actor to replace.
|
||||
ApplyDeployment(command, Sender, isRedeploy: false);
|
||||
ApplyDeployment(command, replyTo, isRedeploy: false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -423,6 +453,76 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
ApplyDeployment(pending.Command, pending.OriginalSender, isRedeploy: true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Notify-and-fetch (Task 10): handles a small central→site
|
||||
/// <see cref="RefreshDeploymentCommand"/>. Fetches the deployment's flattened config
|
||||
/// from central over HTTP via <see cref="IDeploymentConfigFetcher"/>, then pipes the
|
||||
/// result back to self so the existing apply path runs on the actor thread with the
|
||||
/// ORIGINAL sender preserved (the central Ask's temp actor). The reply is the existing
|
||||
/// <see cref="DeploymentStatusResponse"/>, so the central deploy completes unchanged.
|
||||
/// </summary>
|
||||
private void HandleRefreshDeployment(RefreshDeploymentCommand cmd)
|
||||
{
|
||||
// Capture the Ask temp-actor sender BEFORE the async continuation: Akka's Sender
|
||||
// is only valid during synchronous message handling and is no longer the original
|
||||
// sender once the ContinueWith/PipeTo continuation runs on a thread-pool thread.
|
||||
var replyTo = Sender;
|
||||
|
||||
if (_configFetcher is null)
|
||||
{
|
||||
replyTo.Tell(new DeploymentStatusResponse(
|
||||
cmd.DeploymentId, cmd.InstanceUniqueName, DeploymentStatus.Failed,
|
||||
"Deployment config fetcher not available on this node.", DateTimeOffset.UtcNow));
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Fetching config for deployment {DeploymentId} instance {Instance} (notify-and-fetch)",
|
||||
cmd.DeploymentId, cmd.InstanceUniqueName);
|
||||
|
||||
// CancellationToken.None: the fetch is bounded by HttpClient.Timeout. On a singleton
|
||||
// handover mid-fetch the PipeTo lands in dead letters and the central Ask times out
|
||||
// (then reconciles) — acceptable, rare.
|
||||
_configFetcher.FetchAsync(cmd.CentralFetchBaseUrl, cmd.DeploymentId, cmd.FetchToken, CancellationToken.None)
|
||||
.ContinueWith(t => t.IsCompletedSuccessfully
|
||||
? (object)new RefreshFetched(cmd, t.Result, replyTo)
|
||||
: new RefreshFetchFailed(cmd, t.Exception?.GetBaseException().Message ?? "fetch failed", replyTo))
|
||||
.PipeTo(Self);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Notify-and-fetch (Task 10): the config fetch succeeded — reconstruct the in-process
|
||||
/// <see cref="DeployInstanceCommand"/> apply DTO and reuse the existing apply path,
|
||||
/// threading the original central sender through so the
|
||||
/// <see cref="DeploymentStatusResponse"/> reaches it.
|
||||
/// </summary>
|
||||
private void HandleRefreshFetched(RefreshFetched msg)
|
||||
{
|
||||
// Carry the central fetch coordinates on the apply DTO so they survive the whole
|
||||
// apply path (including the PendingRedeploy buffer) down to ApplyDeployment, where
|
||||
// they are replicated to the standby as an id-only notify-and-fetch message.
|
||||
var command = new DeployInstanceCommand(
|
||||
msg.Cmd.DeploymentId, msg.Cmd.InstanceUniqueName, msg.Cmd.RevisionHash,
|
||||
msg.ConfigJson, msg.Cmd.DeployedBy, msg.Cmd.Timestamp,
|
||||
msg.Cmd.CentralFetchBaseUrl, msg.Cmd.FetchToken);
|
||||
HandleDeploy(command, msg.ReplyTo);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Notify-and-fetch (Task 10): the config fetch failed — report
|
||||
/// <see cref="DeploymentStatus.Failed"/> to the original central sender so the deploy
|
||||
/// completes (rather than the central Ask hanging to timeout). Nothing is applied.
|
||||
/// </summary>
|
||||
private void HandleRefreshFetchFailed(RefreshFetchFailed msg)
|
||||
{
|
||||
_logger.LogError(
|
||||
"Config fetch failed for deployment {DeploymentId} instance {Instance}: {Error}",
|
||||
msg.Cmd.DeploymentId, msg.Cmd.InstanceUniqueName, msg.Error);
|
||||
msg.ReplyTo.Tell(new DeploymentStatusResponse(
|
||||
msg.Cmd.DeploymentId, msg.Cmd.InstanceUniqueName, DeploymentStatus.Failed,
|
||||
$"Communication failure: {msg.Error}", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the Instance Actor, persists the config, and replies to the deployer.
|
||||
/// A redeployment is an update of an existing instance, so the deployed-instance
|
||||
@@ -464,10 +564,15 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
await _storage.ClearStaticOverridesAsync(instanceName);
|
||||
await _storage.ClearNativeAlarmsForInstanceAsync(instanceName);
|
||||
|
||||
// Replicate to standby node
|
||||
// Replicate to standby node — notify-and-fetch: send only the deployment id +
|
||||
// central fetch coordinates (NOT the config JSON). The standby fetches the
|
||||
// config over HTTP itself, so a large config never crosses the intra-site Akka
|
||||
// hop (which would silently drop on the 128 KB frame trap). When the coords are
|
||||
// absent (deploy paths other than RefreshDeployment), the standby fetch is a
|
||||
// no-op miss and T18 reconciliation is the durable backstop.
|
||||
_replicationActor?.Tell(new ReplicateConfigDeploy(
|
||||
instanceName, command.FlattenedConfigurationJson,
|
||||
command.DeploymentId, command.RevisionHash, true));
|
||||
instanceName, command.DeploymentId, command.RevisionHash, true,
|
||||
command.CentralFetchBaseUrl ?? "", command.FetchToken ?? ""));
|
||||
|
||||
return new DeployPersistenceResult(
|
||||
command.DeploymentId, instanceName, true, null, sender, isRedeploy);
|
||||
@@ -1590,6 +1695,19 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
internal record PendingRedeploy(DeployInstanceCommand Command, IActorRef OriginalSender);
|
||||
|
||||
/// <summary>
|
||||
/// Notify-and-fetch (Task 10): piped back to self when the deployment's flattened
|
||||
/// config has been fetched from central. Carries the original central sender so the
|
||||
/// reused apply path replies to it.
|
||||
/// </summary>
|
||||
private sealed record RefreshFetched(RefreshDeploymentCommand Cmd, string ConfigJson, IActorRef ReplyTo);
|
||||
|
||||
/// <summary>
|
||||
/// Notify-and-fetch (Task 10): piped back to self when the config fetch failed.
|
||||
/// Carries the original central sender so a Failed status is reported to it.
|
||||
/// </summary>
|
||||
private sealed record RefreshFetchFailed(RefreshDeploymentCommand Cmd, string Error, IActorRef ReplyTo);
|
||||
|
||||
/// <summary>
|
||||
/// SiteRuntime-021: internal message dispatched from
|
||||
/// <see cref="HandleDeployArtifacts"/>'s off-thread persistence task back
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Runs on EVERY site node (NOT a singleton) so a standby that was DOWN during a deploy
|
||||
/// self-heals on its next restart. On startup the actor performs one best-effort
|
||||
/// reconciliation pass:
|
||||
/// <list type="number">
|
||||
/// <item>read the node's local deployed inventory from SQLite,</item>
|
||||
/// <item>report it to central via the <c>SiteCommunicationActor</c> Ask
|
||||
/// (<see cref="ReconcileSiteRequest"/> → <see cref="ReconcileSiteResponse"/>),</item>
|
||||
/// <item>fetch each gap item's config over HTTP and guarded-write it, and</item>
|
||||
/// <item>LOG (never delete) any orphan the node still has but central no longer deploys.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Best-effort throughout.</b> A central-unreachable / timed-out Ask is caught, logged
|
||||
/// at Warning, and the pass simply ends — reconcile re-runs on the next node startup; it is a
|
||||
/// self-heal, not a critical path. A per-item fetch/write failure is caught and logged, then
|
||||
/// the remaining gap items continue (one bad item must not abort the rest). The actor never
|
||||
/// crashes on these failures.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The pass runs after a small startup delay (so the central ClusterClient has time to
|
||||
/// register) and is driven entirely off the actor thread: the Ask + fetch + write happen in
|
||||
/// an awaited continuation whose summary is captured in an internal message
|
||||
/// <see cref="ReconcilePassResult"/> piped back to <c>Self</c>. The actor thread never blocks.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The site does NOT carry the central fetch base URL in its own config — it uses
|
||||
/// <see cref="ReconcileSiteResponse.CentralFetchBaseUrl"/> from central's reply.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
private const string StartupTimerKey = "reconcile-startup";
|
||||
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly IDeploymentConfigFetcher _configFetcher;
|
||||
private readonly IActorRef _siteCommunicationActor;
|
||||
private readonly string _siteIdentifier;
|
||||
private readonly string _nodeId;
|
||||
private readonly ILogger<SiteReconciliationActor> _logger;
|
||||
private readonly TimeSpan _initialDelay;
|
||||
private readonly TimeSpan _askTimeout;
|
||||
|
||||
/// <summary>Akka timer scheduler injected by the framework via <see cref="IWithTimers"/>.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes the per-node startup-reconciliation actor.
|
||||
/// </summary>
|
||||
/// <param name="storage">Site-local SQLite store — read for the inventory, written for the gap.</param>
|
||||
/// <param name="configFetcher">Fetches a deployment's flattened config JSON from central over HTTP.</param>
|
||||
/// <param name="siteCommunicationActor">
|
||||
/// The site's <c>SiteCommunicationActor</c>; it forwards the
|
||||
/// <see cref="ReconcileSiteRequest"/> over the registered central ClusterClient and routes
|
||||
/// the <see cref="ReconcileSiteResponse"/> back to this actor's Ask.
|
||||
/// </param>
|
||||
/// <param name="siteIdentifier">This node's site identifier (resolved by central).</param>
|
||||
/// <param name="nodeId">This node's semantic id (e.g. <c>node-a</c>/<c>node-b</c>), for logging/diagnostics.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
/// <param name="initialDelay">
|
||||
/// Delay before the single startup pass, giving the central ClusterClient time to register.
|
||||
/// Defaults to 5 seconds.
|
||||
/// </param>
|
||||
/// <param name="askTimeout">Round-trip timeout for the reconcile Ask to central. Defaults to 30 seconds.</param>
|
||||
public SiteReconciliationActor(
|
||||
SiteStorageService storage,
|
||||
IDeploymentConfigFetcher configFetcher,
|
||||
IActorRef siteCommunicationActor,
|
||||
string siteIdentifier,
|
||||
string nodeId,
|
||||
ILogger<SiteReconciliationActor> logger,
|
||||
TimeSpan? initialDelay = null,
|
||||
TimeSpan? askTimeout = null)
|
||||
{
|
||||
_storage = storage;
|
||||
_configFetcher = configFetcher;
|
||||
_siteCommunicationActor = siteCommunicationActor;
|
||||
_siteIdentifier = siteIdentifier;
|
||||
_nodeId = nodeId;
|
||||
_logger = logger;
|
||||
_initialDelay = initialDelay ?? TimeSpan.FromSeconds(5);
|
||||
_askTimeout = askTimeout ?? TimeSpan.FromSeconds(30);
|
||||
|
||||
Receive<RunReconcile>(_ => RunReconcilePassAsync().PipeTo(Self));
|
||||
Receive<ReconcilePassResult>(HandleReconcilePassResult);
|
||||
|
||||
// Defensive: RunReconcilePassAsync is designed never to throw (it returns a faulted
|
||||
// ReconcilePassResult instead), but if anything unexpected faults the piped Task the
|
||||
// Status.Failure would otherwise go to dead letters silently. Log it instead.
|
||||
Receive<Status.Failure>(f => _logger.LogWarning(f.Cause,
|
||||
"Reconcile pass faulted unexpectedly for site {Site} node {Node}",
|
||||
_siteIdentifier, _nodeId));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
base.PreStart();
|
||||
// One-shot pass after a small delay so the central ClusterClient can register first.
|
||||
// Non-blocking: the timer fires RunReconcile back onto this actor's mailbox.
|
||||
Timers.StartSingleTimer(StartupTimerKey, RunReconcile.Instance, _initialDelay);
|
||||
_logger.LogInformation(
|
||||
"SiteReconciliationActor started for site {Site} node {Node}; startup reconcile scheduled in {Delay}",
|
||||
_siteIdentifier, _nodeId, _initialDelay);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the full reconcile pass off the actor thread. Never throws: a central-unreachable
|
||||
/// Ask (or any other top-level failure) is captured as a faulted
|
||||
/// <see cref="ReconcilePassResult"/>; per-item fetch/write failures are caught per item so
|
||||
/// the rest of the gap still applies.
|
||||
/// </summary>
|
||||
private async Task<ReconcilePassResult> RunReconcilePassAsync()
|
||||
{
|
||||
Dictionary<string, string> localMap;
|
||||
try
|
||||
{
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync().ConfigureAwait(false);
|
||||
localMap = new Dictionary<string, string>(configs.Count, StringComparer.Ordinal);
|
||||
foreach (var c in configs)
|
||||
localMap[c.InstanceUniqueName] = c.RevisionHash;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return ReconcilePassResult.Faulted(ex);
|
||||
}
|
||||
|
||||
// Report inventory to central and get fresh fetch tokens for the gap. Best-effort:
|
||||
// a central-unreachable / timed-out Ask faults here and is reported as a faulted pass
|
||||
// (logged Warning; reconcile re-runs next startup).
|
||||
ReconcileSiteResponse response;
|
||||
try
|
||||
{
|
||||
response = await _siteCommunicationActor
|
||||
.Ask<ReconcileSiteResponse>(
|
||||
new ReconcileSiteRequest(_siteIdentifier, _nodeId, localMap),
|
||||
_askTimeout)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return ReconcilePassResult.Faulted(ex);
|
||||
}
|
||||
|
||||
var fetched = 0;
|
||||
var failed = 0;
|
||||
|
||||
// Fetch + guarded-write each gap item. Per-item failure must not abort the rest.
|
||||
foreach (var item in response.Gap)
|
||||
{
|
||||
try
|
||||
{
|
||||
var configJson = await _configFetcher
|
||||
.FetchAsync(response.CentralFetchBaseUrl, item.DeploymentId, item.FetchToken, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _storage.StoreDeployedConfigIfNewerAsync(
|
||||
item.InstanceUniqueName, configJson, item.DeploymentId, item.RevisionHash, item.IsEnabled)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
fetched++;
|
||||
_logger.LogInformation(
|
||||
"Reconcile: fetched + stored config for {Instance} (deployment {DeploymentId}, rev {Revision})",
|
||||
item.InstanceUniqueName, item.DeploymentId, item.RevisionHash);
|
||||
}
|
||||
catch (DeploymentConfigFetchException ex) when (ex.IsSuperseded)
|
||||
{
|
||||
// 404 = superseded/expired between staging and fetch; a newer deploy will
|
||||
// replicate it. Not a failure — skip quietly.
|
||||
_logger.LogInformation(
|
||||
"Reconcile: skip {Instance} (deployment {DeploymentId}) — superseded/expired",
|
||||
item.InstanceUniqueName, item.DeploymentId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failed++;
|
||||
_logger.LogError(ex,
|
||||
"Reconcile: failed to fetch/store config for {Instance} (deployment {DeploymentId}) — continuing with remaining items",
|
||||
item.InstanceUniqueName, item.DeploymentId);
|
||||
}
|
||||
}
|
||||
|
||||
// Orphans: present locally but no longer deployed at central. LOG only — never delete
|
||||
// (a stale local row is harmless; deleting risks dropping a config a later deploy needs).
|
||||
foreach (var name in response.OrphanNames)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Reconcile: local instance {Instance} is no longer deployed at central — leaving in place; manual cleanup may be needed",
|
||||
name);
|
||||
}
|
||||
|
||||
return ReconcilePassResult.Completed(fetched, failed, response.OrphanNames.Count);
|
||||
}
|
||||
|
||||
private void HandleReconcilePassResult(ReconcilePassResult result)
|
||||
{
|
||||
if (result.Error != null)
|
||||
{
|
||||
// Best-effort: a failed pass (central unreachable, Ask timeout, local read error) is
|
||||
// logged at Warning and the actor stays alive. Reconcile re-runs on the next startup.
|
||||
_logger.LogWarning(result.Error,
|
||||
"Reconcile pass for site {Site} node {Node} did not complete (central unreachable or read error) — will retry on next startup",
|
||||
_siteIdentifier, _nodeId);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Reconcile pass for site {Site} node {Node} complete: {Fetched} fetched, {Failed} failed, {Orphans} orphan(s)",
|
||||
_siteIdentifier, _nodeId, result.Fetched, result.Failed, result.Orphans);
|
||||
}
|
||||
|
||||
// ── Internal messages ──
|
||||
|
||||
/// <summary>Self-tick that drives the one-shot startup reconcile pass.</summary>
|
||||
private sealed class RunReconcile
|
||||
{
|
||||
public static readonly RunReconcile Instance = new();
|
||||
private RunReconcile() { }
|
||||
}
|
||||
|
||||
/// <summary>Summary of one reconcile pass, piped to <c>Self</c> for logging.</summary>
|
||||
private sealed record ReconcilePassResult(int Fetched, int Failed, int Orphans, Exception? Error)
|
||||
{
|
||||
public static ReconcilePassResult Completed(int fetched, int failed, int orphans)
|
||||
=> new(fetched, failed, orphans, null);
|
||||
|
||||
public static ReconcilePassResult Faulted(Exception error)
|
||||
=> new(0, 0, 0, error);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Event;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
@@ -21,6 +22,7 @@ public class SiteReplicationActor : ReceiveActor
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly StoreAndForwardStorage _sfStorage;
|
||||
private readonly ReplicationService _replicationService;
|
||||
private readonly IDeploymentConfigFetcher? _configFetcher;
|
||||
private readonly string _siteRole;
|
||||
private readonly ILogger<SiteReplicationActor> _logger;
|
||||
private readonly Cluster _cluster;
|
||||
@@ -34,16 +36,24 @@ public class SiteReplicationActor : ReceiveActor
|
||||
/// <param name="replicationService">Service providing replication transport logic.</param>
|
||||
/// <param name="siteRole">Akka cluster role used to identify peer nodes to replicate to.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
/// <param name="configFetcher">
|
||||
/// Fetches a deployed instance's config JSON from central over HTTP. Used by the
|
||||
/// notify-and-fetch standby apply path (<see cref="HandleApplyConfigDeploy"/>): the peer
|
||||
/// replicates only the deployment id, and the standby fetches the config itself so a large
|
||||
/// config never crosses the intra-site Akka hop. Null on nodes/tests without a fetcher.
|
||||
/// </param>
|
||||
public SiteReplicationActor(
|
||||
SiteStorageService storage,
|
||||
StoreAndForwardStorage sfStorage,
|
||||
ReplicationService replicationService,
|
||||
string siteRole,
|
||||
ILogger<SiteReplicationActor> logger)
|
||||
ILogger<SiteReplicationActor> logger,
|
||||
IDeploymentConfigFetcher? configFetcher = null)
|
||||
{
|
||||
_storage = storage;
|
||||
_sfStorage = sfStorage;
|
||||
_replicationService = replicationService;
|
||||
_configFetcher = configFetcher;
|
||||
_siteRole = siteRole;
|
||||
_logger = logger;
|
||||
_cluster = Cluster.Get(Context.System);
|
||||
@@ -55,7 +65,8 @@ public class SiteReplicationActor : ReceiveActor
|
||||
|
||||
// Outbound — forward to peer
|
||||
Receive<ReplicateConfigDeploy>(msg => SendToPeer(new ApplyConfigDeploy(
|
||||
msg.InstanceName, msg.ConfigJson, msg.DeploymentId, msg.RevisionHash, msg.IsEnabled)));
|
||||
msg.InstanceName, msg.DeploymentId, msg.RevisionHash, msg.IsEnabled,
|
||||
msg.CentralFetchBaseUrl, msg.FetchToken)));
|
||||
Receive<ReplicateConfigRemove>(msg => SendToPeer(new ApplyConfigRemove(msg.InstanceName)));
|
||||
Receive<ReplicateConfigSetEnabled>(msg => SendToPeer(new ApplyConfigSetEnabled(
|
||||
msg.InstanceName, msg.IsEnabled)));
|
||||
@@ -120,7 +131,12 @@ public class SiteReplicationActor : ReceiveActor
|
||||
}
|
||||
}
|
||||
|
||||
private void SendToPeer(object message)
|
||||
/// <summary>
|
||||
/// Forwards a replication message to the tracked peer node's <c>site-replication</c> actor
|
||||
/// (fire-and-forget, dropped when no peer is tracked). <see langword="protected virtual"/>
|
||||
/// so tests can intercept the peer send without standing up a real two-node cluster.
|
||||
/// </summary>
|
||||
protected virtual void SendToPeer(object message)
|
||||
{
|
||||
if (_peerAddress == null)
|
||||
{
|
||||
@@ -136,14 +152,68 @@ public class SiteReplicationActor : ReceiveActor
|
||||
|
||||
private void HandleApplyConfigDeploy(ApplyConfigDeploy msg)
|
||||
{
|
||||
_logger.LogInformation("Applying replicated config deploy for {Instance}", msg.InstanceName);
|
||||
_storage.StoreDeployedConfigAsync(
|
||||
msg.InstanceName, msg.ConfigJson, msg.DeploymentId, msg.RevisionHash, msg.IsEnabled)
|
||||
.ContinueWith(t =>
|
||||
if (string.IsNullOrEmpty(msg.CentralFetchBaseUrl))
|
||||
{
|
||||
// The direct DeployInstanceCommand cross-cluster wire path was retired (Task 14).
|
||||
// This guard is a defensive fallback: skip quietly rather than calling FetchAsync("")
|
||||
// and logging a spurious error. T18 reconciliation backstops any missed writes.
|
||||
_logger.LogDebug(
|
||||
"No fetch coords for {Instance} (deployment {DeploymentId}) — skipping replicated fetch; T18 reconciliation is the backstop",
|
||||
msg.InstanceName, msg.DeploymentId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (_configFetcher is null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"No config fetcher available; cannot apply replicated config for {Instance} (deployment {DeploymentId}) — reconciliation will backstop",
|
||||
msg.InstanceName, msg.DeploymentId);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Replicating config for {Instance} (deployment {DeploymentId}) — fetching from central",
|
||||
msg.InstanceName, msg.DeploymentId);
|
||||
|
||||
// Notify-and-fetch: the peer sent only the id, so the standby fetches the config
|
||||
// itself (off-thread; best-effort fire-and-forget, matching the no-ack replication
|
||||
// model). The guarded write only overwrites a strictly-older local row. A single
|
||||
// fetch attempt — T18 reconciliation is the durable backstop for a lost fetch.
|
||||
_configFetcher.FetchAsync(msg.CentralFetchBaseUrl, msg.DeploymentId, msg.FetchToken, CancellationToken.None)
|
||||
.ContinueWith(async t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
_logger.LogError(t.Exception, "Failed to apply replicated deploy for {Instance}", msg.InstanceName);
|
||||
});
|
||||
try
|
||||
{
|
||||
if (t.IsCompletedSuccessfully)
|
||||
{
|
||||
await _storage.StoreDeployedConfigIfNewerAsync(
|
||||
msg.InstanceName, t.Result, msg.DeploymentId, msg.RevisionHash, msg.IsEnabled);
|
||||
return;
|
||||
}
|
||||
|
||||
var ex = t.Exception?.GetBaseException();
|
||||
if (ex is DeploymentConfigFetchException { IsSuperseded: true })
|
||||
_logger.LogInformation(
|
||||
"Skip replicated config for {Instance}: superseded/expired (a newer deploy will replicate)",
|
||||
msg.InstanceName);
|
||||
else if (t.IsCanceled)
|
||||
_logger.LogWarning(
|
||||
"Replicated config fetch cancelled for {Instance} (deployment {DeploymentId})",
|
||||
msg.InstanceName, msg.DeploymentId);
|
||||
else
|
||||
_logger.LogError(ex,
|
||||
"Replicated config fetch failed for {Instance} (deployment {DeploymentId})",
|
||||
msg.InstanceName, msg.DeploymentId);
|
||||
}
|
||||
catch (Exception writeEx)
|
||||
{
|
||||
// Guarded-write failure is best-effort; observe + log so nothing faults silently.
|
||||
_logger.LogError(writeEx,
|
||||
"Failed to write replicated config for {Instance} (deployment {DeploymentId})",
|
||||
msg.InstanceName, msg.DeploymentId);
|
||||
}
|
||||
})
|
||||
.Unwrap();
|
||||
}
|
||||
|
||||
private void HandleApplyConfigRemove(ApplyConfigRemove msg)
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
|
||||
/// <summary>HTTP <see cref="IDeploymentConfigFetcher"/> — GETs the config from central's internal endpoint.</summary>
|
||||
public sealed class HttpDeploymentConfigFetcher : IDeploymentConfigFetcher
|
||||
{
|
||||
private readonly HttpClient _http;
|
||||
private readonly ILogger<HttpDeploymentConfigFetcher> _log;
|
||||
|
||||
public HttpDeploymentConfigFetcher(HttpClient http, ILogger<HttpDeploymentConfigFetcher> log)
|
||||
{
|
||||
_http = http;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
public async Task<string> FetchAsync(string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct)
|
||||
{
|
||||
var url = $"{centralFetchBaseUrl.TrimEnd('/')}/api/internal/deployments/{Uri.EscapeDataString(deploymentId)}/config";
|
||||
using var req = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
req.Headers.Add("X-Deployment-Token", token);
|
||||
|
||||
HttpResponseMessage resp;
|
||||
try
|
||||
{
|
||||
resp = await _http.SendAsync(req, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw; // genuine caller cancellation — propagate
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Includes HttpClient.Timeout (surfaces as TaskCanceledException with ct NOT cancelled) — treat as transport.
|
||||
_log.LogWarning(ex, "deployment-config fetch transport error for {DeploymentId}", deploymentId);
|
||||
throw new DeploymentConfigFetchException($"deployment-config fetch transport error: {ex.Message}", isSuperseded: false, ex);
|
||||
}
|
||||
|
||||
using (resp)
|
||||
{
|
||||
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
|
||||
{
|
||||
_log.LogWarning("deployment-config not found (expired/superseded/unknown) for {DeploymentId} (HTTP 404)", deploymentId);
|
||||
throw new DeploymentConfigFetchException($"deployment-config not found (expired/superseded/unknown): {deploymentId}", isSuperseded: true);
|
||||
}
|
||||
if (!resp.IsSuccessStatusCode)
|
||||
{
|
||||
_log.LogWarning("deployment-config fetch failed for {DeploymentId}: HTTP {StatusCode}", deploymentId, (int)resp.StatusCode);
|
||||
throw new DeploymentConfigFetchException($"deployment-config fetch failed: HTTP {(int)resp.StatusCode}", isSuperseded: false);
|
||||
}
|
||||
|
||||
var body = await resp.Content.ReadAsStringAsync(ct);
|
||||
_log.LogDebug("deployment-config fetched for {DeploymentId} (HTTP {StatusCode})", deploymentId, (int)resp.StatusCode);
|
||||
return body;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// Fetches a deployment's flattened config JSON from central over HTTP
|
||||
/// (notify-and-fetch). The site calls this after receiving a small notify.
|
||||
/// </summary>
|
||||
public interface IDeploymentConfigFetcher
|
||||
{
|
||||
/// <summary>
|
||||
/// GETs the flattened config JSON for <paramref name="deploymentId"/> from
|
||||
/// <paramref name="centralFetchBaseUrl"/>, presenting <paramref name="token"/>
|
||||
/// in the X-Deployment-Token header. Throws <see cref="DeploymentConfigFetchException"/>
|
||||
/// on any non-success; a 404 (expired/superseded/unknown) sets IsSuperseded.
|
||||
/// </summary>
|
||||
Task<string> FetchAsync(string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>Raised when a deployment-config fetch fails. <see cref="IsSuperseded"/> is true on HTTP 404.</summary>
|
||||
public sealed class DeploymentConfigFetchException : Exception
|
||||
{
|
||||
public bool IsSuperseded { get; }
|
||||
public DeploymentConfigFetchException(string message, bool isSuperseded, Exception? inner = null)
|
||||
: base(message, inner) => IsSuperseded = isSuperseded;
|
||||
}
|
||||
@@ -6,9 +6,10 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
|
||||
// Outbound messages — sent by local DeploymentManagerActor/S&F service
|
||||
// to the local SiteReplicationActor for forwarding to the peer node.
|
||||
|
||||
/// <summary>Outbound: replicate a deployed instance config (create or update) to the peer node.</summary>
|
||||
/// <summary>Outbound: tell the peer to fetch+apply a deployed instance config by id (notify-and-fetch; no inline config).</summary>
|
||||
public record ReplicateConfigDeploy(
|
||||
string InstanceName, string ConfigJson, string DeploymentId, string RevisionHash, bool IsEnabled);
|
||||
string InstanceName, string DeploymentId, string RevisionHash, bool IsEnabled,
|
||||
string CentralFetchBaseUrl, string FetchToken);
|
||||
|
||||
/// <summary>Outbound: replicate removal of a deployed instance config to the peer node.</summary>
|
||||
public record ReplicateConfigRemove(string InstanceName);
|
||||
@@ -25,9 +26,10 @@ public record ReplicateStoreAndForward(ReplicationOperation Operation);
|
||||
// Inbound messages — received from the peer's SiteReplicationActor
|
||||
// and applied to local SQLite storage.
|
||||
|
||||
/// <summary>Inbound: apply a peer-replicated instance config (create or update) to local SQLite.</summary>
|
||||
/// <summary>Inbound: peer-replicated config deploy — the standby fetches the config by id and writes it (guarded).</summary>
|
||||
public record ApplyConfigDeploy(
|
||||
string InstanceName, string ConfigJson, string DeploymentId, string RevisionHash, bool IsEnabled);
|
||||
string InstanceName, string DeploymentId, string RevisionHash, bool IsEnabled,
|
||||
string CentralFetchBaseUrl, string FetchToken);
|
||||
|
||||
/// <summary>Inbound: apply peer-replicated removal of a deployed instance config to local SQLite.</summary>
|
||||
public record ApplyConfigRemove(string InstanceName);
|
||||
|
||||
@@ -229,6 +229,71 @@ public class SiteStorageService
|
||||
_logger.LogDebug("Stored deployed config for {Instance}, deploymentId={DeploymentId}", instanceName, deploymentId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stores a deployed instance configuration only if the incoming write is strictly
|
||||
/// newer than any already-stored row for the same instance. If no row exists the
|
||||
/// write is always performed (insert path). Uses SQLite's conditional
|
||||
/// <c>ON CONFLICT … WHERE excluded.deployed_at > deployed_configurations.deployed_at</c>
|
||||
/// clause so the guard is atomic with no application-level read-modify-write.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the standby-node write path for replicated configs. The active-node
|
||||
/// apply path (<see cref="StoreDeployedConfigAsync"/>) remains unguarded and always
|
||||
/// overwrites, because the active node's write is always authoritative.
|
||||
/// <para>
|
||||
/// <paramref name="deployedAtOverride"/> is exposed for testing so that the exact
|
||||
/// <c>deployed_at</c> value can be controlled without sleeping between calls.
|
||||
/// Production callers omit it and receive the default <see cref="DateTimeOffset.UtcNow"/>
|
||||
/// semantics, identical to <see cref="StoreDeployedConfigAsync"/>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="instanceName">The unique name of the instance.</param>
|
||||
/// <param name="configJson">The deployed configuration as JSON.</param>
|
||||
/// <param name="deploymentId">The unique deployment identifier.</param>
|
||||
/// <param name="revisionHash">The configuration revision hash for staleness detection.</param>
|
||||
/// <param name="isEnabled">Whether the instance is enabled.</param>
|
||||
/// <param name="deployedAtOverride">
|
||||
/// Optional explicit <c>deployed_at</c> timestamp. Defaults to
|
||||
/// <see cref="DateTimeOffset.UtcNow"/> when <see langword="null"/>.
|
||||
/// </param>
|
||||
/// <returns>A task that completes when the conditional upsert has been executed.</returns>
|
||||
public async Task StoreDeployedConfigIfNewerAsync(
|
||||
string instanceName,
|
||||
string configJson,
|
||||
string deploymentId,
|
||||
string revisionHash,
|
||||
bool isEnabled,
|
||||
DateTimeOffset? deployedAtOverride = null)
|
||||
{
|
||||
var deployedAt = (deployedAtOverride ?? DateTimeOffset.UtcNow).ToString("O");
|
||||
|
||||
await using var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = @"
|
||||
INSERT INTO deployed_configurations
|
||||
(instance_unique_name, config_json, deployment_id, revision_hash, is_enabled, deployed_at)
|
||||
VALUES (@name, @json, @depId, @hash, @enabled, @deployedAt)
|
||||
ON CONFLICT(instance_unique_name) DO UPDATE SET
|
||||
config_json = excluded.config_json,
|
||||
deployment_id = excluded.deployment_id,
|
||||
revision_hash = excluded.revision_hash,
|
||||
is_enabled = excluded.is_enabled,
|
||||
deployed_at = excluded.deployed_at
|
||||
WHERE excluded.deployed_at > deployed_configurations.deployed_at";
|
||||
|
||||
command.Parameters.AddWithValue("@name", instanceName);
|
||||
command.Parameters.AddWithValue("@json", configJson);
|
||||
command.Parameters.AddWithValue("@depId", deploymentId);
|
||||
command.Parameters.AddWithValue("@hash", revisionHash);
|
||||
command.Parameters.AddWithValue("@enabled", isEnabled ? 1 : 0);
|
||||
command.Parameters.AddWithValue("@deployedAt", deployedAt);
|
||||
|
||||
await command.ExecuteNonQueryAsync();
|
||||
_logger.LogDebug("StoreDeployedConfigIfNewer for {Instance}, deploymentId={DeploymentId}", instanceName, deploymentId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes a deployed instance configuration and its static overrides.
|
||||
/// </summary>
|
||||
|
||||
@@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
@@ -63,6 +64,12 @@ public static class ServiceCollectionExtensions
|
||||
services.AddScoped<IExternalSystemRepository, SiteExternalSystemRepository>();
|
||||
services.AddScoped<INotificationRepository, SiteNotificationRepository>();
|
||||
|
||||
// Notify-and-fetch: typed HttpClient for fetching deployment configs from central.
|
||||
services.AddHttpClient<IDeploymentConfigFetcher, HttpDeploymentConfigFetcher>()
|
||||
.ConfigureHttpClient((sp, c) =>
|
||||
c.Timeout = TimeSpan.FromSeconds(
|
||||
sp.GetRequiredService<IOptions<SiteRuntimeOptions>>().Value.ConfigFetchTimeoutSeconds));
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
|
||||
@@ -56,4 +56,13 @@ public class SiteRuntimeOptions
|
||||
/// Default: 5000ms.
|
||||
/// </summary>
|
||||
public int NativeAlarmRetryIntervalMs { get; set; } = 5000;
|
||||
|
||||
/// <summary>HTTP timeout (seconds) for fetching a deployment config from central (notify-and-fetch).</summary>
|
||||
public int ConfigFetchTimeoutSeconds { get; set; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Bounded retry count for the standby's best-effort replicated-config fetch.
|
||||
/// Reserved — consumed by the standby replication fetch in a later task; not yet wired.
|
||||
/// </summary>
|
||||
public int ConfigFetchRetryCount { get; set; } = 3;
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -58,6 +58,9 @@ public class TopologyPageTests : BunitContext
|
||||
{
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5)
|
||||
}));
|
||||
// Notify-and-fetch (Task 6): DeploymentService now also depends on
|
||||
// IOptions<CommunicationOptions>; register it so the page's DI graph resolves.
|
||||
Services.AddSingleton(Options.Create(new CommunicationOptions()));
|
||||
Services.AddSingleton<ILogger<DeploymentService>>(NullLogger<DeploymentService>.Instance);
|
||||
// DeploymentService gained a DiffService dependency (DeploymentManager
|
||||
// contract change); register it so the page's DI graph resolves.
|
||||
@@ -384,6 +387,93 @@ public class TopologyPageTests : BunitContext
|
||||
Assert.Contains("Changed", markup);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Seeds one site with one deployed (Enabled) instance whose deployment
|
||||
/// comparison resolves: a deployed snapshot plus a current flatten. Used by
|
||||
/// the staleness-lifecycle tests below, which assert on how OFTEN the
|
||||
/// comparison runs (via the snapshot repository call count), not on the
|
||||
/// resulting Stale/Current value.
|
||||
/// </summary>
|
||||
private void SeedOneDeployedInstanceWithComparison()
|
||||
{
|
||||
SeedRepos(
|
||||
sites: new[] { new Site("Plant-A", "plant-a") { Id = 1 } },
|
||||
instances: new[]
|
||||
{
|
||||
new Instance("Pump-001") { Id = 100, SiteId = 1, State = InstanceState.Enabled }
|
||||
});
|
||||
|
||||
var deployedConfig = new FlattenedConfiguration { InstanceUniqueName = "Pump-001" };
|
||||
_deployRepo.GetDeployedSnapshotByInstanceIdAsync(100, Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<DeployedConfigSnapshot?>(
|
||||
new DeployedConfigSnapshot("dep-1", "hash-old",
|
||||
JsonSerializer.Serialize(deployedConfig))));
|
||||
|
||||
var currentConfig = new FlattenedConfiguration { InstanceUniqueName = "Pump-001" };
|
||||
_pipeline.FlattenAndValidateAsync(100, Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult(Result<FlatteningPipelineResult>.Success(
|
||||
new FlatteningPipelineResult(currentConfig, "hash-new", ValidationResult.Success()))));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LiveUpdate_DoesNotRecomputeStaleness()
|
||||
{
|
||||
// Performance regression guard: staleness (the expensive per-instance
|
||||
// re-flatten via DeploymentService.GetDeploymentComparisonAsync) must NOT
|
||||
// run on the 15s live-update poll. The live-update path refreshes only the
|
||||
// cheap deployed state (LoadDeployedStateAsync), which the timer invokes
|
||||
// directly. The comparison reaches into the snapshot repository, so its
|
||||
// call count is the proxy for "did staleness recompute".
|
||||
SeedOneDeployedInstanceWithComparison();
|
||||
|
||||
var cut = Render<TopologyPage>();
|
||||
|
||||
// Initial load computed staleness exactly once.
|
||||
await _deployRepo.Received(1)
|
||||
.GetDeployedSnapshotByInstanceIdAsync(100, Arg.Any<CancellationToken>());
|
||||
|
||||
// Drive the EXACT code path the live-update timer runs.
|
||||
await cut.InvokeAsync(() => cut.Instance.LoadDeployedStateAsync());
|
||||
|
||||
// Staleness was NOT recomputed by the live-update refresh — still once.
|
||||
await _deployRepo.Received(1)
|
||||
.GetDeployedSnapshotByInstanceIdAsync(100, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeployedState_Renders_AfterSplit()
|
||||
{
|
||||
// After splitting the load into fast-state vs expensive-staleness, the
|
||||
// cheap deployed state (hierarchy + State badge) must still render.
|
||||
SeedOneDeployedInstanceWithComparison();
|
||||
|
||||
var cut = Render<TopologyPage>();
|
||||
FindToggleForLabel(cut, "Plant-A")!.Click();
|
||||
|
||||
Assert.Contains("Pump-001", cut.Markup);
|
||||
Assert.Contains("Enabled", cut.Markup);
|
||||
// The deployed instance shows a staleness badge (Stale or Current).
|
||||
Assert.Matches("Stale|Current", cut.Markup);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManualRecheck_RecomputesStaleness()
|
||||
{
|
||||
// The "Re-check staleness" button lets operators refresh staleness on
|
||||
// demand (since it's off the live loop). Clicking it runs the comparison
|
||||
// again — a second snapshot-repository call.
|
||||
SeedOneDeployedInstanceWithComparison();
|
||||
|
||||
var cut = Render<TopologyPage>();
|
||||
_deployRepo.Received(1)
|
||||
.GetDeployedSnapshotByInstanceIdAsync(100, Arg.Any<CancellationToken>());
|
||||
|
||||
cut.Find("button[aria-label='Re-check staleness']").Click();
|
||||
|
||||
_deployRepo.Received(2)
|
||||
.GetDeployedSnapshotByInstanceIdAsync(100, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LegacyInstancesRoute_IsDeclaredOnTopologyPage()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Tests;
|
||||
|
||||
public class DeploymentFetchTokenTests
|
||||
{
|
||||
[Fact]
|
||||
public void Generate_ReturnsUrlSafeTokenOfAdequateLength()
|
||||
{
|
||||
var token = DeploymentFetchToken.Generate();
|
||||
|
||||
Assert.True(token.Length >= 32, $"Expected length >= 32 but got {token.Length}");
|
||||
Assert.DoesNotContain('+', token);
|
||||
Assert.DoesNotContain('/', token);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Generate_TwoCallsReturnDifferentValues()
|
||||
{
|
||||
var a = DeploymentFetchToken.Generate();
|
||||
var b = DeploymentFetchToken.Generate();
|
||||
|
||||
Assert.NotEqual(a, b);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConstantTimeEquals_SameToken_ReturnsTrue()
|
||||
{
|
||||
var token = DeploymentFetchToken.Generate();
|
||||
|
||||
Assert.True(DeploymentFetchToken.ConstantTimeEquals(token, token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConstantTimeEquals_TokenWithSuffix_ReturnsFalse()
|
||||
{
|
||||
var token = DeploymentFetchToken.Generate();
|
||||
|
||||
Assert.False(DeploymentFetchToken.ConstantTimeEquals(token, token + "x"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConstantTimeEquals_BothEmpty_ReturnsTrue()
|
||||
{
|
||||
Assert.True(DeploymentFetchToken.ConstantTimeEquals("", ""));
|
||||
}
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests that <see cref="CentralCommunicationActor"/> routes a site→central
|
||||
/// <see cref="ReconcileSiteRequest"/> through the scoped <see cref="ReconcileService"/>
|
||||
/// and pipes the resulting <see cref="ReconcileSiteResponse"/> back to the original
|
||||
/// sender (the site's ClusterClient path). Mirrors the audit-ingest routing tests.
|
||||
/// </summary>
|
||||
public class CentralCommunicationActorReconcileTests : TestKit
|
||||
{
|
||||
[Fact]
|
||||
public void ReconcileSiteRequest_RoutesResponseToSender()
|
||||
{
|
||||
var deploymentRepo = Substitute.For<IDeploymentManagerRepository>();
|
||||
var siteRepo = Substitute.For<ISiteRepository>();
|
||||
|
||||
// GetAllSitesAsync is called by the actor's periodic refresh; keep it empty.
|
||||
siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new List<Site>());
|
||||
|
||||
siteRepo.GetSiteByIdentifierAsync("site1", Arg.Any<CancellationToken>())
|
||||
.Returns(new Site("Site One", "site1") { Id = 7 });
|
||||
|
||||
deploymentRepo.GetExpectedDeploymentsForSiteAsync(7, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<ExpectedDeployment>
|
||||
{
|
||||
new(2, "inst-B", "rev2", "dep-B", true),
|
||||
});
|
||||
deploymentRepo.GetDeployedSnapshotByInstanceIdAsync(2, Arg.Any<CancellationToken>())
|
||||
.Returns(new DeployedConfigSnapshot("dep-B", "rev2", "{\"cfg\":\"B\"}"));
|
||||
deploymentRepo.StagePendingIfAbsentAsync(
|
||||
Arg.Any<int>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(),
|
||||
Arg.Any<string>(), Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(),
|
||||
Arg.Any<CancellationToken>())
|
||||
.Returns(true);
|
||||
|
||||
var options = Options.Create(new CommunicationOptions
|
||||
{
|
||||
CentralFetchBaseUrl = "https://central.example:9000",
|
||||
PendingDeploymentTtl = TimeSpan.FromMinutes(5),
|
||||
});
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => deploymentRepo);
|
||||
services.AddScoped(_ => siteRepo);
|
||||
services.AddSingleton(options);
|
||||
services.AddSingleton<Microsoft.Extensions.Logging.ILogger<ReconcileService>>(
|
||||
NullLogger<ReconcileService>.Instance);
|
||||
services.AddScoped<ReconcileService>();
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var factory = Substitute.For<ISiteClientFactory>();
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, factory, null)));
|
||||
|
||||
// Node B is missing inst-B entirely → it should come back as a gap item.
|
||||
actor.Tell(new ReconcileSiteRequest(
|
||||
"site1", "node-b",
|
||||
new Dictionary<string, string>()));
|
||||
|
||||
var response = ExpectMsg<ReconcileSiteResponse>(TimeSpan.FromSeconds(5));
|
||||
var gap = Assert.Single(response.Gap);
|
||||
Assert.Equal("inst-B", gap.InstanceUniqueName);
|
||||
Assert.Equal("dep-B", gap.DeploymentId);
|
||||
Assert.False(string.IsNullOrWhiteSpace(gap.FetchToken));
|
||||
}
|
||||
}
|
||||
@@ -82,23 +82,22 @@ public class CentralCommunicationActorTests : TestKit
|
||||
new("Test Site", identifier) { NodeAAddress = nodeAAddress, NodeBAddress = nodeBAddress };
|
||||
|
||||
[Fact]
|
||||
public void ClusterClientRouting_RoutesToConfiguredSite()
|
||||
public void ClusterClientRouting_RefreshDeploymentCommand_RoutesToSite()
|
||||
{
|
||||
var site = CreateSite("site1", "akka.tcp://scadabridge@host:8082");
|
||||
var (actor, _, siteProbes) = CreateActorWithMockRepo(new[] { site });
|
||||
|
||||
// Wait for auto-refresh (PreStart schedules with TimeSpan.Zero initial delay)
|
||||
Thread.Sleep(1000);
|
||||
|
||||
var command = new DeployInstanceCommand(
|
||||
"dep1", "inst1", "hash1", "{}", "admin", DateTimeOffset.UtcNow);
|
||||
var command = new RefreshDeploymentCommand(
|
||||
"dep1", "inst1", "rev1", "admin", DateTimeOffset.UtcNow,
|
||||
"https://central:9000", "tok1");
|
||||
actor.Tell(new SiteEnvelope("site1", command));
|
||||
|
||||
// The site1 probe (acting as ClusterClient) should receive a ClusterClient.Send
|
||||
var msg = siteProbes["site1"].ExpectMsg<ClusterClient.Send>();
|
||||
Assert.Equal("/user/site-communication", msg.Path);
|
||||
Assert.IsType<DeployInstanceCommand>(msg.Message);
|
||||
Assert.Equal("dep1", ((DeployInstanceCommand)msg.Message).DeploymentId);
|
||||
Assert.IsType<RefreshDeploymentCommand>(msg.Message);
|
||||
Assert.Equal("dep1", ((RefreshDeploymentCommand)msg.Message).DeploymentId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -109,8 +108,9 @@ public class CentralCommunicationActorTests : TestKit
|
||||
// Wait for auto-refresh
|
||||
Thread.Sleep(1000);
|
||||
|
||||
var command = new DeployInstanceCommand(
|
||||
"dep1", "inst1", "hash1", "{}", "admin", DateTimeOffset.UtcNow);
|
||||
var command = new RefreshDeploymentCommand(
|
||||
"dep1", "inst1", "hash1", "admin", DateTimeOffset.UtcNow,
|
||||
"https://central:9000", "tok1");
|
||||
actor.Tell(new SiteEnvelope("unknown-site", command));
|
||||
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
@@ -158,11 +158,12 @@ public class CentralCommunicationActorTests : TestKit
|
||||
Thread.Sleep(1000);
|
||||
|
||||
// Verify routing to site1 works
|
||||
var cmd1 = new DeployInstanceCommand(
|
||||
"dep1", "inst1", "hash1", "{}", "admin", DateTimeOffset.UtcNow);
|
||||
var cmd1 = new RefreshDeploymentCommand(
|
||||
"dep1", "inst1", "hash1", "admin", DateTimeOffset.UtcNow,
|
||||
"https://central:9000", "tok1");
|
||||
actor.Tell(new SiteEnvelope("site1", cmd1));
|
||||
var msg1 = siteProbes["site1"].ExpectMsg<ClusterClient.Send>();
|
||||
Assert.Equal("dep1", ((DeployInstanceCommand)msg1.Message).DeploymentId);
|
||||
Assert.Equal("dep1", ((RefreshDeploymentCommand)msg1.Message).DeploymentId);
|
||||
|
||||
// Update mock repo to return both sites
|
||||
var site2 = CreateSite("site2", "akka.tcp://scadabridge@host2:8082");
|
||||
@@ -174,11 +175,12 @@ public class CentralCommunicationActorTests : TestKit
|
||||
Thread.Sleep(1000);
|
||||
|
||||
// Verify routing to site2 now works
|
||||
var cmd2 = new DeployInstanceCommand(
|
||||
"dep2", "inst2", "hash2", "{}", "admin", DateTimeOffset.UtcNow);
|
||||
var cmd2 = new RefreshDeploymentCommand(
|
||||
"dep2", "inst2", "hash2", "admin", DateTimeOffset.UtcNow,
|
||||
"https://central:9000", "tok2");
|
||||
actor.Tell(new SiteEnvelope("site2", cmd2));
|
||||
var msg2 = siteProbes["site2"].ExpectMsg<ClusterClient.Send>();
|
||||
Assert.Equal("dep2", ((DeployInstanceCommand)msg2.Message).DeploymentId);
|
||||
Assert.Equal("dep2", ((RefreshDeploymentCommand)msg2.Message).DeploymentId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -225,14 +227,15 @@ public class CentralCommunicationActorTests : TestKit
|
||||
Thread.Sleep(1000);
|
||||
|
||||
// good-site must still be registered and routable despite bad-site failing to parse.
|
||||
var cmd = new DeployInstanceCommand(
|
||||
"dep1", "inst1", "hash1", "{}", "admin", DateTimeOffset.UtcNow);
|
||||
var cmd = new RefreshDeploymentCommand(
|
||||
"dep1", "inst1", "hash1", "admin", DateTimeOffset.UtcNow,
|
||||
"https://central:9000", "tok1");
|
||||
actor.Tell(new SiteEnvelope("good-site", cmd));
|
||||
|
||||
Assert.True(siteProbes.ContainsKey("good-site"),
|
||||
"good-site should have a ClusterClient even though bad-site's address is malformed");
|
||||
var msg = siteProbes["good-site"].ExpectMsg<ClusterClient.Send>();
|
||||
Assert.Equal("dep1", ((DeployInstanceCommand)msg.Message).DeploymentId);
|
||||
Assert.Equal("dep1", ((RefreshDeploymentCommand)msg.Message).DeploymentId);
|
||||
}
|
||||
|
||||
private NotificationSubmit CreateSubmit(string id = "notif1") =>
|
||||
|
||||
@@ -15,20 +15,6 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
/// </summary>
|
||||
public class CommunicationServiceTests : TestKit
|
||||
{
|
||||
[Fact]
|
||||
public async Task BeforeInitialization_ThrowsOnUsage()
|
||||
{
|
||||
var options = Options.Create(new CommunicationOptions());
|
||||
var logger = NullLogger<CommunicationService>.Instance;
|
||||
var service = new CommunicationService(options, logger);
|
||||
|
||||
// CommunicationService requires SetCommunicationActor before use
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
service.DeployInstanceAsync("site1",
|
||||
new DeployInstanceCommand(
|
||||
"dep1", "inst1", "hash1", "{}", "admin", DateTimeOffset.UtcNow)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnsubscribeDebugView_IsTellNotAsk()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="PendingDeploymentPurgeActor"/> — the notify-and-fetch central
|
||||
/// singleton that sweeps expired PendingDeployment staging rows. The actor is thin glue
|
||||
/// over <c>IDeploymentManagerRepository.PurgeExpiredPendingDeploymentsAsync</c> (whose
|
||||
/// row-level semantics are covered by the repository tests); these tests pin the actor's
|
||||
/// own policy: it opens a per-tick DI scope, calls the purge with a now-ish threshold,
|
||||
/// and — being best-effort hygiene — survives a throwing purge and keeps ticking.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The substitute's return is driven by a closure with an <see cref="System.Threading.Interlocked"/>
|
||||
/// counter rather than NSubstitute's <c>ReceivedCalls()</c>, because the repeating
|
||||
/// scheduler timer records calls on a background thread concurrently with the test
|
||||
/// thread's assertions — querying NSubstitute's call list under that concurrency can
|
||||
/// throw "collection was modified". A fenced counter is safe to read while the timer runs.
|
||||
/// </remarks>
|
||||
public class PendingDeploymentPurgeActorTests : TestKit
|
||||
{
|
||||
private IActorRef CreateActor(IDeploymentManagerRepository repo, TimeSpan interval)
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
// Mirror AddConfigurationDatabase: IDeploymentManagerRepository is scoped, so the
|
||||
// actor opens a fresh scope per tick and resolves the repository there.
|
||||
services.AddScoped(_ => repo);
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var options = Options.Create(new CommunicationOptions { PendingDeploymentPurgeInterval = interval });
|
||||
return Sys.ActorOf(Props.Create(() => new PendingDeploymentPurgeActor(
|
||||
sp, options, NullLogger<PendingDeploymentPurgeActor>.Instance)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Tick_PurgesExpiredRows_WithNowThreshold()
|
||||
{
|
||||
var calls = 0;
|
||||
DateTimeOffset captured = default;
|
||||
var repo = Substitute.For<IDeploymentManagerRepository>();
|
||||
repo.PurgeExpiredPendingDeploymentsAsync(Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
captured = ci.Arg<DateTimeOffset>();
|
||||
Interlocked.Increment(ref calls);
|
||||
return 3;
|
||||
});
|
||||
|
||||
// Large interval so the PreStart timer never fires during the test — only the
|
||||
// single manual tick below drives a purge, making the count deterministic.
|
||||
var actor = CreateActor(repo, TimeSpan.FromHours(1));
|
||||
|
||||
var before = DateTimeOffset.UtcNow;
|
||||
actor.Tell(PendingDeploymentPurgeActor.PurgeTick.Instance);
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(1, Volatile.Read(ref calls)),
|
||||
duration: TimeSpan.FromSeconds(3),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
|
||||
// The purge threshold is "now" — bounded by the window around the tick. Reading
|
||||
// `captured` after the fenced counter read is safe (Interlocked establishes the
|
||||
// happens-before with the write that precedes the increment).
|
||||
var after = DateTimeOffset.UtcNow;
|
||||
Assert.InRange(captured, before, after);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Tick_PurgeThrows_ActorSurvives_AndRetriesOnNextInterval()
|
||||
{
|
||||
var calls = 0;
|
||||
var repo = Substitute.For<IDeploymentManagerRepository>();
|
||||
repo.PurgeExpiredPendingDeploymentsAsync(Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||||
.Returns<int>(_ =>
|
||||
{
|
||||
Interlocked.Increment(ref calls);
|
||||
throw new InvalidOperationException("simulated purge failure");
|
||||
});
|
||||
|
||||
// Fast interval so the self-scheduling timer fires repeatedly; the actor must
|
||||
// swallow each per-tick exception and keep ticking (best-effort hygiene — a
|
||||
// crash here would drop the singleton until failover).
|
||||
CreateActor(repo, TimeSpan.FromMilliseconds(100));
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.True(Volatile.Read(ref calls) >= 2,
|
||||
$"expected >= 2 ticks despite failures, got {Volatile.Read(ref calls)}"),
|
||||
duration: TimeSpan.FromSeconds(5),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for the central-side startup-reconciliation handler
|
||||
/// (<see cref="ReconcileService"/>). A site node reports its local inventory; the
|
||||
/// service diffs it against central's expected deployed set, stages fresh fetch
|
||||
/// tokens for the gap (missing/stale), and reports orphans.
|
||||
/// </summary>
|
||||
public class ReconcileServiceTests
|
||||
{
|
||||
private const string BaseUrl = "https://central.example:9000";
|
||||
private const string SiteIdentifier = "site1";
|
||||
private const int SiteId = 7;
|
||||
|
||||
private readonly IDeploymentManagerRepository _deploymentRepo =
|
||||
Substitute.For<IDeploymentManagerRepository>();
|
||||
private readonly ISiteRepository _siteRepo = Substitute.For<ISiteRepository>();
|
||||
|
||||
private ReconcileService CreateService(TimeSpan? ttl = null)
|
||||
{
|
||||
var options = Options.Create(new CommunicationOptions
|
||||
{
|
||||
CentralFetchBaseUrl = BaseUrl,
|
||||
PendingDeploymentTtl = ttl ?? TimeSpan.FromMinutes(5),
|
||||
});
|
||||
return new ReconcileService(
|
||||
_deploymentRepo, _siteRepo, options, NullLogger<ReconcileService>.Instance);
|
||||
}
|
||||
|
||||
private void SiteResolves() =>
|
||||
_siteRepo.GetSiteByIdentifierAsync(SiteIdentifier, Arg.Any<CancellationToken>())
|
||||
.Returns(new Site("Site One", SiteIdentifier) { Id = SiteId });
|
||||
|
||||
private static ExpectedDeployment Expected(int id, string name, string rev, string dep, bool enabled = true) =>
|
||||
new(id, name, rev, dep, enabled);
|
||||
|
||||
private void ExpectedSet(params ExpectedDeployment[] expected) =>
|
||||
_deploymentRepo.GetExpectedDeploymentsForSiteAsync(SiteId, Arg.Any<CancellationToken>())
|
||||
.Returns(expected.ToList());
|
||||
|
||||
private void SnapshotFor(ExpectedDeployment exp) =>
|
||||
_deploymentRepo.GetDeployedSnapshotByInstanceIdAsync(exp.InstanceId, Arg.Any<CancellationToken>())
|
||||
.Returns(new DeployedConfigSnapshot(exp.DeploymentId, exp.RevisionHash, $"{{\"cfg\":\"{exp.InstanceUniqueName}\"}}"));
|
||||
|
||||
private void StageReturns(bool result) =>
|
||||
_deploymentRepo.StagePendingIfAbsentAsync(
|
||||
Arg.Any<int>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(),
|
||||
Arg.Any<string>(), Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(),
|
||||
Arg.Any<CancellationToken>())
|
||||
.Returns(result);
|
||||
|
||||
private static ReconcileSiteRequest Request(params (string Name, string Rev)[] local) =>
|
||||
new(SiteIdentifier, "node-a",
|
||||
local.ToDictionary(x => x.Name, x => x.Rev));
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_GapIsMissingAndStale_CurrentOmitted_WithFreshTokensAndSnapshotDeploymentIds()
|
||||
{
|
||||
SiteResolves();
|
||||
var a = Expected(1, "inst-A", "rev1", "dep-A");
|
||||
var b = Expected(2, "inst-B", "rev2", "dep-B");
|
||||
var c = Expected(3, "inst-C", "rev3", "dep-C", enabled: false);
|
||||
ExpectedSet(a, b, c);
|
||||
SnapshotFor(b);
|
||||
SnapshotFor(c);
|
||||
StageReturns(true);
|
||||
|
||||
// Node has A current, B stale (revOLD), C missing entirely.
|
||||
var response = await CreateService().ReconcileAsync(
|
||||
Request(("inst-A", "rev1"), ("inst-B", "revOLD")));
|
||||
|
||||
Assert.Equal(BaseUrl, response.CentralFetchBaseUrl);
|
||||
Assert.Equal(2, response.Gap.Count);
|
||||
Assert.DoesNotContain(response.Gap, g => g.InstanceUniqueName == "inst-A");
|
||||
|
||||
var gapB = Assert.Single(response.Gap, g => g.InstanceUniqueName == "inst-B");
|
||||
Assert.Equal("dep-B", gapB.DeploymentId);
|
||||
Assert.Equal("rev2", gapB.RevisionHash);
|
||||
Assert.True(gapB.IsEnabled);
|
||||
Assert.False(string.IsNullOrWhiteSpace(gapB.FetchToken));
|
||||
|
||||
var gapC = Assert.Single(response.Gap, g => g.InstanceUniqueName == "inst-C");
|
||||
Assert.Equal("dep-C", gapC.DeploymentId);
|
||||
Assert.Equal("rev3", gapC.RevisionHash);
|
||||
Assert.False(gapC.IsEnabled);
|
||||
Assert.False(string.IsNullOrWhiteSpace(gapC.FetchToken));
|
||||
|
||||
// Fresh, distinct tokens per gap item.
|
||||
Assert.NotEqual(gapB.FetchToken, gapC.FetchToken);
|
||||
Assert.Empty(response.OrphanNames);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_LocalNameNotInExpected_IsReportedAsOrphan()
|
||||
{
|
||||
SiteResolves();
|
||||
var a = Expected(1, "inst-A", "rev1", "dep-A");
|
||||
ExpectedSet(a);
|
||||
StageReturns(true);
|
||||
|
||||
// inst-A is current; inst-Z is not deployed centrally → orphan.
|
||||
var response = await CreateService().ReconcileAsync(
|
||||
Request(("inst-A", "rev1"), ("inst-Z", "revX")));
|
||||
|
||||
Assert.Empty(response.Gap);
|
||||
var orphan = Assert.Single(response.OrphanNames);
|
||||
Assert.Equal("inst-Z", orphan);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_StagePendingReturnsFalse_ExistingRowPresent_ReturnsExistingTokenInGap()
|
||||
{
|
||||
// Live-found bug: two site nodes concurrently missing the SAME instance both reconcile at
|
||||
// startup. The first stages a pending row; the second's StagePendingIfAbsent returns false.
|
||||
// The second node must STILL heal — the handler reads the existing pending row and returns
|
||||
// ITS deploymentId/revHash/token (the fetch token is multi-use within its TTL).
|
||||
SiteResolves();
|
||||
var b = Expected(2, "inst-B", "rev2", "dep-B");
|
||||
var c = Expected(3, "inst-C", "rev3", "dep-C");
|
||||
ExpectedSet(b, c);
|
||||
SnapshotFor(b);
|
||||
SnapshotFor(c);
|
||||
|
||||
// B stages fresh (true). C already has a pending row (a concurrent reconcile from the
|
||||
// other node, or an in-flight deploy, staged it first) → stage returns false.
|
||||
StageReturns(true);
|
||||
_deploymentRepo.StagePendingIfAbsentAsync(
|
||||
3, Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(),
|
||||
Arg.Any<string>(), Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(),
|
||||
Arg.Any<CancellationToken>())
|
||||
.Returns(false);
|
||||
|
||||
// The existing pending row for inst-C carries the FIRST node's deploymentId + token.
|
||||
var nowUtc = DateTimeOffset.UtcNow;
|
||||
_deploymentRepo.GetPendingDeploymentByInstanceIdAsync(3, Arg.Any<DateTimeOffset?>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new PendingDeployment(
|
||||
"dep-C-existing", 3, "rev3-existing", "{\"cfg\":\"inst-C\"}",
|
||||
"tok-C-existing", nowUtc, nowUtc.AddMinutes(5)));
|
||||
|
||||
var response = await CreateService().ReconcileAsync(Request(/* empty local inventory */));
|
||||
|
||||
Assert.Equal(2, response.Gap.Count);
|
||||
|
||||
var gapB = Assert.Single(response.Gap, g => g.InstanceUniqueName == "inst-B");
|
||||
Assert.Equal("dep-B", gapB.DeploymentId);
|
||||
|
||||
// inst-C IS included, carrying the EXISTING row's deploymentId/revHash/token — NOT the
|
||||
// snapshot's (snapshot.DeploymentId would be "dep-C").
|
||||
var gapC = Assert.Single(response.Gap, g => g.InstanceUniqueName == "inst-C");
|
||||
Assert.Equal("dep-C-existing", gapC.DeploymentId);
|
||||
Assert.Equal("rev3-existing", gapC.RevisionHash);
|
||||
Assert.Equal("tok-C-existing", gapC.FetchToken);
|
||||
Assert.True(gapC.IsEnabled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_StagePendingReturnsFalse_NoExistingRow_OmitsThatGapItem()
|
||||
{
|
||||
// Fallback path: stage returns false, but the pending row raced away (was purged between
|
||||
// the stage attempt and the read). Omit the item — the node retries on the next reconcile.
|
||||
SiteResolves();
|
||||
var b = Expected(2, "inst-B", "rev2", "dep-B");
|
||||
var c = Expected(3, "inst-C", "rev3", "dep-C");
|
||||
ExpectedSet(b, c);
|
||||
SnapshotFor(b);
|
||||
SnapshotFor(c);
|
||||
|
||||
StageReturns(true);
|
||||
_deploymentRepo.StagePendingIfAbsentAsync(
|
||||
3, Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(),
|
||||
Arg.Any<string>(), Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(),
|
||||
Arg.Any<CancellationToken>())
|
||||
.Returns(false);
|
||||
|
||||
// The pending row was purged between the stage attempt and the read.
|
||||
_deploymentRepo.GetPendingDeploymentByInstanceIdAsync(3, Arg.Any<DateTimeOffset?>(), Arg.Any<CancellationToken>())
|
||||
.Returns((PendingDeployment?)null);
|
||||
|
||||
var response = await CreateService().ReconcileAsync(Request(/* empty local inventory */));
|
||||
|
||||
var gap = Assert.Single(response.Gap);
|
||||
Assert.Equal("inst-B", gap.InstanceUniqueName);
|
||||
Assert.DoesNotContain(response.Gap, g => g.InstanceUniqueName == "inst-C");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_SnapshotMissing_SkipsGapItem()
|
||||
{
|
||||
SiteResolves();
|
||||
var b = Expected(2, "inst-B", "rev2", "dep-B");
|
||||
ExpectedSet(b);
|
||||
// No snapshot configured for inst-B → repo returns null (deleted race).
|
||||
StageReturns(true);
|
||||
|
||||
var response = await CreateService().ReconcileAsync(Request());
|
||||
|
||||
Assert.Empty(response.Gap);
|
||||
Assert.Empty(response.OrphanNames);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_UnknownSite_ReturnsEmptyResponse_NoThrow()
|
||||
{
|
||||
_siteRepo.GetSiteByIdentifierAsync(SiteIdentifier, Arg.Any<CancellationToken>())
|
||||
.Returns((Site?)null);
|
||||
|
||||
var response = await CreateService().ReconcileAsync(
|
||||
Request(("inst-A", "rev1")));
|
||||
|
||||
Assert.Empty(response.Gap);
|
||||
Assert.Empty(response.OrphanNames);
|
||||
Assert.Equal(BaseUrl, response.CentralFetchBaseUrl);
|
||||
await _deploymentRepo.DidNotReceive()
|
||||
.GetExpectedDeploymentsForSiteAsync(Arg.Any<int>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
}
|
||||
@@ -26,17 +26,18 @@ public class SiteCommunicationActorTests : TestKit
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeployCommand_ForwardedToDeploymentManager()
|
||||
public void RefreshDeploymentCommand_ForwardedToDeploymentManager()
|
||||
{
|
||||
var dmProbe = CreateTestProbe();
|
||||
var siteActor = Sys.ActorOf(Props.Create(() =>
|
||||
new SiteCommunicationActor("site1", _options, dmProbe.Ref)));
|
||||
|
||||
var command = new DeployInstanceCommand(
|
||||
"dep1", "inst1", "hash1", "{}", "admin", DateTimeOffset.UtcNow);
|
||||
var command = new RefreshDeploymentCommand(
|
||||
"dep1", "inst1", "hash1", "admin", DateTimeOffset.UtcNow,
|
||||
"https://central/", "tok1");
|
||||
siteActor.Tell(command);
|
||||
|
||||
dmProbe.ExpectMsg<DeployInstanceCommand>(msg => msg.DeploymentId == "dep1");
|
||||
dmProbe.ExpectMsg<RefreshDeploymentCommand>(msg => msg.DeploymentId == "dep1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+319
@@ -0,0 +1,319 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Coverage for the notify-and-fetch <see cref="PendingDeployment"/> staging store on
|
||||
/// <see cref="DeploymentManagerRepository"/>: supersession (a newer deploy for the same
|
||||
/// instance replaces the prior pending row), multi-instance coexistence, TTL purge, and
|
||||
/// delete-by-id. Uses the shared SQLite in-memory fixture, which enforces the
|
||||
/// PendingDeployment → Instance FK, so each test seeds real Instance rows.
|
||||
/// </summary>
|
||||
public class PendingDeploymentRepositoryTests : IDisposable
|
||||
{
|
||||
private readonly ScadaBridgeDbContext _context;
|
||||
private readonly DeploymentManagerRepository _repository;
|
||||
|
||||
public PendingDeploymentRepositoryTests()
|
||||
{
|
||||
_context = SqliteTestHelper.CreateInMemoryContext();
|
||||
_repository = new DeploymentManagerRepository(_context);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_context.Database.CloseConnection();
|
||||
_context.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Seeds an Instance (with its required Site + Template) and returns its generated id.
|
||||
/// SQLite EnsureCreated enforces the FK to Instances, so a real row is required.
|
||||
/// </summary>
|
||||
private async Task<int> SeedInstanceAsync(string uniqueName)
|
||||
{
|
||||
var site = new Site($"Site-{uniqueName}", $"S-{uniqueName}");
|
||||
var template = new Template($"T-{uniqueName}");
|
||||
_context.Sites.Add(site);
|
||||
_context.Templates.Add(template);
|
||||
await _context.SaveChangesAsync();
|
||||
|
||||
var instance = new Instance(uniqueName) { SiteId = site.Id, TemplateId = template.Id };
|
||||
_context.Instances.Add(instance);
|
||||
await _context.SaveChangesAsync();
|
||||
return instance.Id;
|
||||
}
|
||||
|
||||
private static PendingDeployment NewPending(string deploymentId, int instanceId, string config)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
return new PendingDeployment(
|
||||
deploymentId, instanceId, revisionHash: "rev-" + deploymentId,
|
||||
configurationJson: config, token: "tok-" + deploymentId,
|
||||
createdAtUtc: now, expiresAtUtc: now.AddMinutes(10));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AddPendingDeployment_SecondDeployForSameInstance_SupersedesPriorRow()
|
||||
{
|
||||
var instanceId = await SeedInstanceAsync("Inst7");
|
||||
|
||||
await _repository.AddPendingDeploymentAsync(NewPending("dep1", instanceId, "{\"v\":1}"));
|
||||
await _repository.SaveChangesAsync();
|
||||
|
||||
await _repository.AddPendingDeploymentAsync(NewPending("dep2", instanceId, "{\"v\":2}"));
|
||||
await _repository.SaveChangesAsync();
|
||||
|
||||
// The prior pending row must be gone; only the newest deploy survives.
|
||||
Assert.Null(await _repository.GetPendingDeploymentByIdAsync("dep1"));
|
||||
|
||||
var current = await _repository.GetPendingDeploymentByIdAsync("dep2");
|
||||
Assert.NotNull(current);
|
||||
Assert.Equal("{\"v\":2}", current!.ConfigurationJson);
|
||||
Assert.Equal(instanceId, current.InstanceId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AddPendingDeployment_DifferentInstances_Coexist()
|
||||
{
|
||||
var instance1 = await SeedInstanceAsync("Inst1");
|
||||
var instance2 = await SeedInstanceAsync("Inst2");
|
||||
|
||||
await _repository.AddPendingDeploymentAsync(NewPending("depA", instance1, "{\"a\":1}"));
|
||||
await _repository.AddPendingDeploymentAsync(NewPending("depB", instance2, "{\"b\":1}"));
|
||||
await _repository.SaveChangesAsync();
|
||||
|
||||
var a = await _repository.GetPendingDeploymentByIdAsync("depA");
|
||||
var b = await _repository.GetPendingDeploymentByIdAsync("depB");
|
||||
Assert.NotNull(a);
|
||||
Assert.NotNull(b);
|
||||
Assert.Equal(instance1, a!.InstanceId);
|
||||
Assert.Equal(instance2, b!.InstanceId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PurgeExpiredPendingDeployments_RemovesOnlyExpired_ReturnsCount()
|
||||
{
|
||||
var instance1 = await SeedInstanceAsync("Inst1");
|
||||
var instance2 = await SeedInstanceAsync("Inst2");
|
||||
var instance3 = await SeedInstanceAsync("Inst3");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Two expired rows (ExpiresAtUtc <= now) and one live row (in the future).
|
||||
await _repository.AddPendingDeploymentAsync(new PendingDeployment(
|
||||
"expired-1", instance1, "rev", "{}", "tok", now.AddMinutes(-20), now.AddMinutes(-10)));
|
||||
await _repository.SaveChangesAsync();
|
||||
await _repository.AddPendingDeploymentAsync(new PendingDeployment(
|
||||
"expired-2", instance2, "rev", "{}", "tok", now.AddMinutes(-20), now));
|
||||
await _repository.SaveChangesAsync();
|
||||
await _repository.AddPendingDeploymentAsync(new PendingDeployment(
|
||||
"live-1", instance3, "rev", "{}", "tok", now, now.AddMinutes(10)));
|
||||
await _repository.SaveChangesAsync();
|
||||
|
||||
var purged = await _repository.PurgeExpiredPendingDeploymentsAsync(now);
|
||||
|
||||
Assert.Equal(2, purged);
|
||||
Assert.Null(await _repository.GetPendingDeploymentByIdAsync("expired-1"));
|
||||
Assert.Null(await _repository.GetPendingDeploymentByIdAsync("expired-2"));
|
||||
Assert.NotNull(await _repository.GetPendingDeploymentByIdAsync("live-1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PurgeExpiredPendingDeployments_NoneExpired_ReturnsZero()
|
||||
{
|
||||
var instanceId = await SeedInstanceAsync("Inst1");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
await _repository.AddPendingDeploymentAsync(new PendingDeployment(
|
||||
"live-1", instanceId, "rev", "{}", "tok", now, now.AddMinutes(10)));
|
||||
await _repository.SaveChangesAsync();
|
||||
|
||||
var purged = await _repository.PurgeExpiredPendingDeploymentsAsync(now);
|
||||
|
||||
Assert.Equal(0, purged);
|
||||
Assert.NotNull(await _repository.GetPendingDeploymentByIdAsync("live-1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeletePendingDeploymentById_RemovesRow()
|
||||
{
|
||||
var instanceId = await SeedInstanceAsync("Inst1");
|
||||
await _repository.AddPendingDeploymentAsync(NewPending("dep1", instanceId, "{}"));
|
||||
await _repository.SaveChangesAsync();
|
||||
|
||||
await _repository.DeletePendingDeploymentByIdAsync("dep1");
|
||||
await _repository.SaveChangesAsync();
|
||||
|
||||
Assert.Null(await _repository.GetPendingDeploymentByIdAsync("dep1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeletePendingDeploymentById_UnknownId_NoThrow()
|
||||
{
|
||||
await _repository.DeletePendingDeploymentByIdAsync("does-not-exist");
|
||||
await _repository.SaveChangesAsync();
|
||||
Assert.Null(await _repository.GetPendingDeploymentByIdAsync("does-not-exist"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingDeploymentByInstanceId_ReturnsRow_WhenPresent()
|
||||
{
|
||||
// Startup-reconcile read path: when StagePendingIfAbsent reports a row already exists
|
||||
// (concurrent reconcile from the other node, or an in-flight deploy), the handler reads
|
||||
// the existing row by instance id to hand the second node the same fetch token.
|
||||
var instanceId = await SeedInstanceAsync("Inst9");
|
||||
await _repository.AddPendingDeploymentAsync(NewPending("dep9", instanceId, "{\"v\":9}"));
|
||||
await _repository.SaveChangesAsync();
|
||||
|
||||
var row = await _repository.GetPendingDeploymentByInstanceIdAsync(instanceId);
|
||||
|
||||
Assert.NotNull(row);
|
||||
Assert.Equal("dep9", row!.DeploymentId);
|
||||
Assert.Equal(instanceId, row.InstanceId);
|
||||
Assert.Equal("tok-dep9", row.Token);
|
||||
Assert.Equal("{\"v\":9}", row.ConfigurationJson);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingDeploymentByInstanceId_ReturnsNull_WhenAbsent()
|
||||
{
|
||||
// No pending row staged for this instance → null (the reconcile fallback/omit path).
|
||||
var instanceId = await SeedInstanceAsync("Inst10");
|
||||
|
||||
var row = await _repository.GetPendingDeploymentByInstanceIdAsync(instanceId);
|
||||
|
||||
Assert.Null(row);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StagePendingIfAbsent_ExpiredRowDifferentDeploymentId_DoesNotBlock_RemovesExpiredAndStagesLiveRow()
|
||||
{
|
||||
// Live-found bug: pending rows are only TTL-purged (the periodic purge is a deferred TODO),
|
||||
// so an EXPIRED-but-unpurged row for the instance must NOT make staging report "a deploy is
|
||||
// in flight" — it would hand the node an expired token (HTTP 404) and leave it unhealed.
|
||||
// The expired row is removed and the fresh, live row is staged.
|
||||
var instanceId = await SeedInstanceAsync("InstExp1");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Seed an EXPIRED pending row (different deploymentId), bypassing the repo.
|
||||
_context.Set<PendingDeployment>().Add(new PendingDeployment(
|
||||
"dep-expired", instanceId, "rev-old", "{\"old\":true}", "tok-old",
|
||||
now.AddMinutes(-20), now.AddMinutes(-10)));
|
||||
await _context.SaveChangesAsync();
|
||||
|
||||
var staged = await _repository.StagePendingIfAbsentAsync(
|
||||
instanceId, "dep-fresh", "rev-fresh", "{\"fresh\":true}", "tok-fresh",
|
||||
now, now.AddMinutes(10));
|
||||
|
||||
Assert.True(staged);
|
||||
Assert.Null(await _repository.GetPendingDeploymentByIdAsync("dep-expired"));
|
||||
|
||||
var row = await _repository.GetPendingDeploymentByIdAsync("dep-fresh");
|
||||
Assert.NotNull(row);
|
||||
Assert.Equal(instanceId, row!.InstanceId);
|
||||
Assert.Equal("tok-fresh", row.Token);
|
||||
Assert.True(row.ExpiresAtUtc > now); // the new row is live
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StagePendingIfAbsent_ExpiredRowSameDeploymentId_NoUniqueCollision_StagesNew()
|
||||
{
|
||||
// A fresh reconcile re-stages the snapshot's own DeploymentId. If an EXPIRED row already
|
||||
// carries that same DeploymentId, staging must remove it FIRST to avoid colliding on the
|
||||
// PendingDeployment.DeploymentId UNIQUE index — re-staging "D1" over an expired "D1" works.
|
||||
var instanceId = await SeedInstanceAsync("InstExp2");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
_context.Set<PendingDeployment>().Add(new PendingDeployment(
|
||||
"D1", instanceId, "rev-old", "{\"old\":true}", "tok-old",
|
||||
now.AddMinutes(-20), now.AddMinutes(-10)));
|
||||
await _context.SaveChangesAsync();
|
||||
|
||||
var staged = await _repository.StagePendingIfAbsentAsync(
|
||||
instanceId, "D1", "rev-new", "{\"new\":true}", "tok-new",
|
||||
now, now.AddMinutes(10));
|
||||
|
||||
Assert.True(staged);
|
||||
var row = await _repository.GetPendingDeploymentByIdAsync("D1");
|
||||
Assert.NotNull(row);
|
||||
Assert.Equal("tok-new", row!.Token);
|
||||
Assert.Equal("{\"new\":true}", row.ConfigurationJson);
|
||||
Assert.True(row.ExpiresAtUtc > now);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StagePendingIfAbsent_LiveRow_Blocks_LeavesUntouched()
|
||||
{
|
||||
// A still-LIVE pending row (future expiry) signals a genuine in-flight deploy or a concurrent
|
||||
// reconcile — staging must return false and leave it untouched (no supersession).
|
||||
var instanceId = await SeedInstanceAsync("InstLive");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
_context.Set<PendingDeployment>().Add(new PendingDeployment(
|
||||
"dep-live", instanceId, "rev-live", "{\"live\":true}", "tok-live",
|
||||
now.AddMinutes(-1), now.AddMinutes(9)));
|
||||
await _context.SaveChangesAsync();
|
||||
|
||||
var staged = await _repository.StagePendingIfAbsentAsync(
|
||||
instanceId, "dep-new", "rev-new", "{\"new\":true}", "tok-new",
|
||||
now, now.AddMinutes(10));
|
||||
|
||||
Assert.False(staged);
|
||||
var existing = await _repository.GetPendingDeploymentByIdAsync("dep-live");
|
||||
Assert.NotNull(existing);
|
||||
Assert.Equal("tok-live", existing!.Token);
|
||||
Assert.Null(await _repository.GetPendingDeploymentByIdAsync("dep-new"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingDeploymentByInstanceId_WithNowUtc_ReturnsLiveRow_NotExpired()
|
||||
{
|
||||
// The by-instance getter must never hand back an EXPIRED row (whose token the config-fetch
|
||||
// endpoint would 404). With nowUtc supplied, an expired row is filtered out; a live row is returned.
|
||||
var expiredInst = await SeedInstanceAsync("InstGetExpired");
|
||||
var liveInst = await SeedInstanceAsync("InstGetLive");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
_context.Set<PendingDeployment>().Add(new PendingDeployment(
|
||||
"dep-getexp", expiredInst, "rev", "{}", "tok-exp",
|
||||
now.AddMinutes(-20), now.AddMinutes(-10)));
|
||||
_context.Set<PendingDeployment>().Add(new PendingDeployment(
|
||||
"dep-getlive", liveInst, "rev", "{}", "tok-live",
|
||||
now.AddMinutes(-1), now.AddMinutes(9)));
|
||||
await _context.SaveChangesAsync();
|
||||
|
||||
// Expired row filtered out when nowUtc is supplied.
|
||||
Assert.Null(await _repository.GetPendingDeploymentByInstanceIdAsync(expiredInst, now));
|
||||
// Live row still returned.
|
||||
var live = await _repository.GetPendingDeploymentByInstanceIdAsync(liveInst, now);
|
||||
Assert.NotNull(live);
|
||||
Assert.Equal("dep-getlive", live!.DeploymentId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AddPendingDeployment_MultiplePriorRowsSameInstance_AllSuperseded()
|
||||
{
|
||||
// Guards the defensive ToListAsync + RemoveRange supersession path: if a prior
|
||||
// bug or corruption left >1 pending row for an instance, a new deploy must clear
|
||||
// them all (a refactor to FirstOrDefault + Remove would silently regress this).
|
||||
var instanceId = await SeedInstanceAsync("Inst8");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Force two pending rows for the same instance, bypassing the repo, to simulate corruption.
|
||||
_context.Set<PendingDeployment>().AddRange(
|
||||
new PendingDeployment("stale-a", instanceId, "rev", "{}", "tok", now.AddMinutes(-5), now.AddMinutes(5)),
|
||||
new PendingDeployment("stale-b", instanceId, "rev", "{}", "tok", now.AddMinutes(-3), now.AddMinutes(7)));
|
||||
await _context.SaveChangesAsync();
|
||||
|
||||
await _repository.AddPendingDeploymentAsync(NewPending("dep3", instanceId, "{\"v\":3}"));
|
||||
await _repository.SaveChangesAsync();
|
||||
|
||||
Assert.Null(await _repository.GetPendingDeploymentByIdAsync("stale-a"));
|
||||
Assert.Null(await _repository.GetPendingDeploymentByIdAsync("stale-b"));
|
||||
Assert.NotNull(await _repository.GetPendingDeploymentByIdAsync("dep3"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Coverage for the startup-reconciliation repo methods on <see cref="DeploymentManagerRepository"/>:
|
||||
/// <see cref="IDeploymentManagerRepository.GetExpectedDeploymentsForSiteAsync"/> (central's deployed
|
||||
/// set for a site, excluding instances with no snapshot) and
|
||||
/// <see cref="IDeploymentManagerRepository.StagePendingIfAbsentAsync"/> (insert-if-absent, used by
|
||||
/// the reconcile handler to mint fresh fetch tokens for the gap without clobbering an in-flight deploy).
|
||||
/// Uses the shared SQLite in-memory fixture with real FK enforcement.
|
||||
/// </summary>
|
||||
public class ReconcileRepositoryTests : IDisposable
|
||||
{
|
||||
private readonly ScadaBridgeDbContext _context;
|
||||
private readonly DeploymentManagerRepository _repository;
|
||||
|
||||
public ReconcileRepositoryTests()
|
||||
{
|
||||
_context = SqliteTestHelper.CreateInMemoryContext();
|
||||
_repository = new DeploymentManagerRepository(_context);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_context.Database.CloseConnection();
|
||||
_context.Dispose();
|
||||
}
|
||||
|
||||
// --- Seed helpers ---
|
||||
|
||||
/// <summary>Seeds a Site row and returns its generated id.</summary>
|
||||
private async Task<int> SeedSiteAsync(string name)
|
||||
{
|
||||
var site = new Site(name, $"SI-{name}");
|
||||
_context.Sites.Add(site);
|
||||
await _context.SaveChangesAsync();
|
||||
return site.Id;
|
||||
}
|
||||
|
||||
/// <summary>Seeds a Template + Instance on the given site and returns the instance id.</summary>
|
||||
private async Task<int> SeedInstanceAsync(string uniqueName, int siteId, InstanceState state = InstanceState.Enabled)
|
||||
{
|
||||
var template = new Template($"T-{uniqueName}");
|
||||
_context.Templates.Add(template);
|
||||
await _context.SaveChangesAsync();
|
||||
|
||||
var instance = new Instance(uniqueName) { SiteId = siteId, TemplateId = template.Id, State = state };
|
||||
_context.Instances.Add(instance);
|
||||
await _context.SaveChangesAsync();
|
||||
return instance.Id;
|
||||
}
|
||||
|
||||
/// <summary>Seeds a <see cref="DeployedConfigSnapshot"/> for the given instance and returns it.</summary>
|
||||
private async Task<DeployedConfigSnapshot> SeedSnapshotAsync(int instanceId, string deploymentId, string revisionHash)
|
||||
{
|
||||
var snapshot = new DeployedConfigSnapshot(deploymentId, revisionHash, "{}") { InstanceId = instanceId };
|
||||
_context.Set<DeployedConfigSnapshot>().Add(snapshot);
|
||||
await _context.SaveChangesAsync();
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
// --- GetExpectedDeploymentsForSiteAsync ---
|
||||
|
||||
[Fact]
|
||||
public async Task GetExpectedDeployments_ReturnsOnlyDeployedInstancesForSite()
|
||||
{
|
||||
// Arrange: site A has two deployed instances (one Enabled, one Disabled)
|
||||
// site B has one deployed instance
|
||||
// site A also has one instance WITHOUT a snapshot (must be excluded)
|
||||
var siteAId = await SeedSiteAsync("SiteA");
|
||||
var siteBId = await SeedSiteAsync("SiteB");
|
||||
|
||||
var instA1Id = await SeedInstanceAsync("A-Inst1", siteAId, InstanceState.Enabled);
|
||||
var instA2Id = await SeedInstanceAsync("A-Inst2", siteAId, InstanceState.Disabled);
|
||||
var instA3Id = await SeedInstanceAsync("A-Inst3-NoSnap", siteAId, InstanceState.Enabled);
|
||||
var instB1Id = await SeedInstanceAsync("B-Inst1", siteBId, InstanceState.Enabled);
|
||||
|
||||
await SeedSnapshotAsync(instA1Id, "dep-a1", "hash-a1");
|
||||
await SeedSnapshotAsync(instA2Id, "dep-a2", "hash-a2");
|
||||
// instA3 intentionally has no snapshot
|
||||
await SeedSnapshotAsync(instB1Id, "dep-b1", "hash-b1");
|
||||
|
||||
// Act
|
||||
var result = await _repository.GetExpectedDeploymentsForSiteAsync(siteAId);
|
||||
|
||||
// Assert: exactly the two site-A instances that have snapshots
|
||||
Assert.Equal(2, result.Count);
|
||||
|
||||
var byName = result.ToDictionary(r => r.InstanceUniqueName);
|
||||
Assert.True(byName.ContainsKey("A-Inst1"), "A-Inst1 should be present");
|
||||
Assert.True(byName.ContainsKey("A-Inst2"), "A-Inst2 should be present");
|
||||
Assert.False(byName.ContainsKey("A-Inst3-NoSnap"), "instance without snapshot must be excluded");
|
||||
Assert.False(byName.ContainsKey("B-Inst1"), "site B instance must be excluded");
|
||||
|
||||
// Check field correctness for A-Inst1 (Enabled)
|
||||
var a1 = byName["A-Inst1"];
|
||||
Assert.Equal(instA1Id, a1.InstanceId);
|
||||
Assert.Equal("dep-a1", a1.DeploymentId);
|
||||
Assert.Equal("hash-a1", a1.RevisionHash);
|
||||
Assert.True(a1.IsEnabled);
|
||||
|
||||
// Check field correctness for A-Inst2 (Disabled)
|
||||
var a2 = byName["A-Inst2"];
|
||||
Assert.Equal(instA2Id, a2.InstanceId);
|
||||
Assert.Equal("dep-a2", a2.DeploymentId);
|
||||
Assert.Equal("hash-a2", a2.RevisionHash);
|
||||
Assert.False(a2.IsEnabled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetExpectedDeployments_NoDeployedInstances_ReturnsEmpty()
|
||||
{
|
||||
var siteId = await SeedSiteAsync("EmptySite");
|
||||
// Seed an instance but no snapshot
|
||||
await SeedInstanceAsync("Inst-NoSnap", siteId);
|
||||
|
||||
var result = await _repository.GetExpectedDeploymentsForSiteAsync(siteId);
|
||||
|
||||
Assert.Empty(result);
|
||||
}
|
||||
|
||||
// --- StagePendingIfAbsentAsync ---
|
||||
|
||||
[Fact]
|
||||
public async Task StagePendingIfAbsent_NoPriorRow_StagesAndReturnsTrue()
|
||||
{
|
||||
// Arrange
|
||||
var siteId = await SeedSiteAsync("SiteX");
|
||||
var instanceId = await SeedInstanceAsync("InstX", siteId);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Act
|
||||
var staged = await _repository.StagePendingIfAbsentAsync(
|
||||
instanceId,
|
||||
deploymentId: "dep-new",
|
||||
revisionHash: "rev-new",
|
||||
configurationJson: "{\"x\":1}",
|
||||
token: "tok-new",
|
||||
createdAtUtc: now,
|
||||
expiresAtUtc: now.AddMinutes(10));
|
||||
|
||||
// Assert: staged, and the row is retrievable with the correct token + config
|
||||
Assert.True(staged);
|
||||
var row = await _repository.GetPendingDeploymentByIdAsync("dep-new");
|
||||
Assert.NotNull(row);
|
||||
Assert.Equal(instanceId, row!.InstanceId);
|
||||
Assert.Equal("tok-new", row.Token);
|
||||
Assert.Equal("{\"x\":1}", row.ConfigurationJson);
|
||||
Assert.Equal("rev-new", row.RevisionHash);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StagePendingIfAbsent_PriorRowExists_ReturnsFalseAndDoesNotOverwrite()
|
||||
{
|
||||
// Arrange: an existing pending row for the instance (simulates an in-flight deploy)
|
||||
var siteId = await SeedSiteAsync("SiteY");
|
||||
var instanceId = await SeedInstanceAsync("InstY", siteId);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
var existing = new PendingDeployment(
|
||||
"dep-existing", instanceId, revisionHash: "rev-existing",
|
||||
configurationJson: "{\"existing\":true}", token: "tok-existing",
|
||||
createdAtUtc: now.AddMinutes(-5), expiresAtUtc: now.AddMinutes(5));
|
||||
_context.Set<PendingDeployment>().Add(existing);
|
||||
await _context.SaveChangesAsync();
|
||||
|
||||
// Act: try to stage a newer row for the same instance
|
||||
var staged = await _repository.StagePendingIfAbsentAsync(
|
||||
instanceId,
|
||||
deploymentId: "dep-reconcile",
|
||||
revisionHash: "rev-reconcile",
|
||||
configurationJson: "{\"reconcile\":true}",
|
||||
token: "tok-reconcile",
|
||||
createdAtUtc: now,
|
||||
expiresAtUtc: now.AddMinutes(10));
|
||||
|
||||
// Assert: skipped (returns false), existing row is unchanged
|
||||
Assert.False(staged);
|
||||
|
||||
var existingRow = await _repository.GetPendingDeploymentByIdAsync("dep-existing");
|
||||
Assert.NotNull(existingRow);
|
||||
Assert.Equal("tok-existing", existingRow!.Token);
|
||||
Assert.Equal("{\"existing\":true}", existingRow.ConfigurationJson);
|
||||
|
||||
// The reconcile row must not exist
|
||||
Assert.Null(await _repository.GetPendingDeploymentByIdAsync("dep-reconcile"));
|
||||
}
|
||||
}
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// SQL Server integration coverage for the notify-and-fetch <see cref="PendingDeployment"/>
|
||||
/// staging store's <b>same-DeploymentId re-stage</b> path. The SQLite in-memory fixture
|
||||
/// (<c>PendingDeploymentRepositoryTests</c>) covers the logic, but the delete-before-insert
|
||||
/// ordering that lets <see cref="DeploymentManagerRepository.StagePendingIfAbsentAsync"/>
|
||||
/// re-stage an instance's OWN <c>DeploymentId</c> over an expired row depends on EF emitting
|
||||
/// the DELETE before the INSERT within a single <c>SaveChanges</c> — against the production
|
||||
/// UNIQUE index on <c>DeploymentId</c>. SQLite's constraint timing differs from SQL Server's
|
||||
/// (SQLite defers/relaxes within a transaction where SQL Server enforces per-statement), so
|
||||
/// this class asserts the behaviour against the real migrated schema via
|
||||
/// <see cref="MsSqlMigrationFixture"/> rather than the SQLite provider.
|
||||
/// </summary>
|
||||
public class PendingDeploymentRepositoryIntegrationTests : IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
public PendingDeploymentRepositoryIntegrationTests(MsSqlMigrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task StagePendingIfAbsent_ExpiredRowSameDeploymentId_ReStages_NoUniqueViolation()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var instanceId = await SeedInstanceAsync("ReStageInst");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Seed an EXPIRED pending row carrying DeploymentId "D1". A startup reconcile re-stages
|
||||
// the DeployedConfigSnapshot's OWN DeploymentId, so the fresh row reuses "D1" — the exact
|
||||
// collision the unique index would reject if the DELETE didn't precede the INSERT.
|
||||
await using (var seedContext = CreateContext())
|
||||
{
|
||||
seedContext.Set<PendingDeployment>().Add(new PendingDeployment(
|
||||
"D1", instanceId, "rev-old", "{\"old\":true}", "tok-old",
|
||||
now.AddMinutes(-20), now.AddMinutes(-10)));
|
||||
await seedContext.SaveChangesAsync();
|
||||
}
|
||||
|
||||
// Re-stage "D1" over the expired row. Against the real UNIQUE index on DeploymentId this
|
||||
// MUST succeed (EF orders delete-before-insert in the single SaveChanges) rather than
|
||||
// throwing SqlException 2627/2601.
|
||||
await using (var context = CreateContext())
|
||||
{
|
||||
var repo = new DeploymentManagerRepository(context);
|
||||
var staged = await repo.StagePendingIfAbsentAsync(
|
||||
instanceId, "D1", "rev-new", "{\"new\":true}", "tok-new",
|
||||
now, now.AddMinutes(10));
|
||||
Assert.True(staged);
|
||||
}
|
||||
|
||||
// Exactly one row for "D1" survives — the fresh one.
|
||||
await using (var readContext = CreateContext())
|
||||
{
|
||||
var rows = await readContext.Set<PendingDeployment>()
|
||||
.Where(p => p.DeploymentId == "D1")
|
||||
.ToListAsync();
|
||||
Assert.Single(rows);
|
||||
Assert.Equal("tok-new", rows[0].Token);
|
||||
Assert.Equal("{\"new\":true}", rows[0].ConfigurationJson);
|
||||
Assert.True(rows[0].ExpiresAtUtc > now);
|
||||
}
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task AddPendingDeployment_NewDeployDifferentId_SupersedesPriorRow_OnRealSqlServer()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var instanceId = await SeedInstanceAsync("SupersedeInst");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
await using (var context = CreateContext())
|
||||
{
|
||||
var repo = new DeploymentManagerRepository(context);
|
||||
await repo.AddPendingDeploymentAsync(new PendingDeployment(
|
||||
"dep-A", instanceId, "rev-a", "{\"v\":1}", "tok-a", now, now.AddMinutes(5)));
|
||||
await repo.SaveChangesAsync();
|
||||
}
|
||||
|
||||
// A newer deploy for the SAME instance with a DIFFERENT DeploymentId must replace the
|
||||
// prior row (delete-then-insert under the per-instance operation lock) with no FK or
|
||||
// unique conflict on real SQL Server.
|
||||
await using (var context = CreateContext())
|
||||
{
|
||||
var repo = new DeploymentManagerRepository(context);
|
||||
await repo.AddPendingDeploymentAsync(new PendingDeployment(
|
||||
"dep-B", instanceId, "rev-b", "{\"v\":2}", "tok-b", now, now.AddMinutes(5)));
|
||||
await repo.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await using (var readContext = CreateContext())
|
||||
{
|
||||
var rows = await readContext.Set<PendingDeployment>()
|
||||
.Where(p => p.InstanceId == instanceId)
|
||||
.ToListAsync();
|
||||
Assert.Single(rows);
|
||||
Assert.Equal("dep-B", rows[0].DeploymentId);
|
||||
Assert.Equal("{\"v\":2}", rows[0].ConfigurationJson);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Seeds an Instance (with its required Site + Template) against the fixture database and
|
||||
/// returns its generated id. The PendingDeployment → Instance FK requires a real parent row.
|
||||
/// </summary>
|
||||
private async Task<int> SeedInstanceAsync(string uniqueName)
|
||||
{
|
||||
await using var context = CreateContext();
|
||||
var site = new Site($"Site-{uniqueName}", $"S-{uniqueName}");
|
||||
var template = new Template($"T-{uniqueName}");
|
||||
context.Sites.Add(site);
|
||||
context.Templates.Add(template);
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
var instance = new Instance(uniqueName) { SiteId = site.Id, TemplateId = template.Id };
|
||||
context.Instances.Add(instance);
|
||||
await context.SaveChangesAsync();
|
||||
return instance.Id;
|
||||
}
|
||||
|
||||
private ScadaBridgeDbContext CreateContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.Options;
|
||||
return new ScadaBridgeDbContext(options);
|
||||
}
|
||||
}
|
||||
@@ -54,9 +54,35 @@ public class DeploymentServiceTests : TestKit
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
options,
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Notify-and-fetch (Task 6): the central HTTP base URL carried in the
|
||||
/// <c>RefreshDeploymentCommand</c>. Must be non-empty for the deploy path to
|
||||
/// clear the fail-fast configuration guard.
|
||||
/// </summary>
|
||||
private const string TestFetchBaseUrl = "https://central.test:9000";
|
||||
|
||||
/// <summary>How long a staged <c>PendingDeployment</c> stays valid in these tests.</summary>
|
||||
private static readonly TimeSpan TestPendingTtl = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>
|
||||
/// Builds the <see cref="CommunicationOptions"/> the notify-and-fetch deploy
|
||||
/// path reads: a non-empty <see cref="CommunicationOptions.CentralFetchBaseUrl"/>
|
||||
/// (so the fail-fast guard passes) and a fixed
|
||||
/// <see cref="CommunicationOptions.PendingDeploymentTtl"/>. Pass an empty base
|
||||
/// URL to exercise the fail-fast path.
|
||||
/// </summary>
|
||||
private static IOptions<CommunicationOptions> DeployCommOptions(
|
||||
string centralFetchBaseUrl = TestFetchBaseUrl) =>
|
||||
Options.Create(new CommunicationOptions
|
||||
{
|
||||
CentralFetchBaseUrl = centralFetchBaseUrl,
|
||||
PendingDeploymentTtl = TestPendingTtl
|
||||
});
|
||||
|
||||
// ── WP-1: Deployment flow ──
|
||||
|
||||
[Fact]
|
||||
@@ -139,6 +165,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }),
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
|
||||
var result = await service.DeployInstanceAsync(99, "admin");
|
||||
@@ -322,6 +349,63 @@ public class DeploymentServiceTests : TestKit
|
||||
Arg.Is<CancellationToken>(ct => !ct.IsCancellationRequested));
|
||||
}
|
||||
|
||||
// ── AskTimeoutException: Akka's AskTimeoutException must be classified as a deploy timeout ──
|
||||
|
||||
[Fact]
|
||||
public async Task DeployInstanceAsync_AskTimeoutException_ClassifiedAsTimeout()
|
||||
{
|
||||
// Arrange: first-time deploy with an actor that drops every message, so
|
||||
// the RefreshDeploymentAsync Ask never receives a reply and throws
|
||||
// AskTimeoutException (which does NOT derive from System.TimeoutException).
|
||||
// Prior to the fix this fell through to the generic "Deployment error:"
|
||||
// branch instead of the timeout "Communication failure:" branch, which
|
||||
// broke the DeploymentManager-006 reconciliation that keys off that prefix.
|
||||
var instance = new Instance("AskTimeoutInst") { Id = 99, SiteId = 1, State = InstanceState.NotDeployed };
|
||||
_repo.GetInstanceByIdAsync(99, Arg.Any<CancellationToken>()).Returns(instance);
|
||||
SetupValidPipeline(99, "AskTimeoutInst", "sha256:target");
|
||||
_repo.GetCurrentDeploymentStatusAsync(99, Arg.Any<CancellationToken>())
|
||||
.Returns((DeploymentRecord?)null);
|
||||
|
||||
DeploymentRecord? captured = null;
|
||||
await _repo.AddDeploymentRecordAsync(
|
||||
Arg.Do<DeploymentRecord>(r => captured = r), Arg.Any<CancellationToken>());
|
||||
|
||||
// A 50 ms DeploymentTimeout forces AskTimeoutException immediately
|
||||
// when the actor silently drops every message (never replies).
|
||||
var droppingActor = Sys.ActorOf(Props.Create<DroppingActor>());
|
||||
var comms = new CommunicationService(
|
||||
Options.Create(new CommunicationOptions { DeploymentTimeout = TimeSpan.FromMilliseconds(50) }),
|
||||
NullLogger<CommunicationService>.Instance);
|
||||
comms.SetCommunicationActor(droppingActor);
|
||||
|
||||
var siteRepo = CreateSiteRepoStub();
|
||||
var service = new DeploymentService(
|
||||
_repo, siteRepo, _pipeline, comms, _lockManager, _audit,
|
||||
new DiffService(),
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }),
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
|
||||
// Act
|
||||
var result = await service.DeployInstanceAsync(99, "admin");
|
||||
|
||||
// Assert: AskTimeoutException must route through the isTimeout branch.
|
||||
Assert.True(result.IsFailure);
|
||||
// Result error must reflect a timeout, not a generic deploy failure.
|
||||
Assert.StartsWith("Deployment timed out:", result.Error);
|
||||
Assert.DoesNotContain("Deployment failed:", result.Error);
|
||||
|
||||
// The deployment record must be Failed with the timeout-failure prefix
|
||||
// (which the DeploymentManager-006 reconciliation check keys off).
|
||||
Assert.NotNull(captured);
|
||||
Assert.Equal(DeploymentStatus.Failed, captured!.Status);
|
||||
Assert.NotNull(captured.ErrorMessage);
|
||||
Assert.StartsWith("Communication failure:", captured.ErrorMessage);
|
||||
Assert.DoesNotContain("Deployment error:", captured.ErrorMessage);
|
||||
}
|
||||
|
||||
// ── WP-6: Lifecycle commands ──
|
||||
|
||||
[Fact]
|
||||
@@ -924,9 +1008,13 @@ public class DeploymentServiceTests : TestKit
|
||||
|
||||
/// <summary>
|
||||
/// Builds a DeploymentService whose CommunicationService is backed by the
|
||||
/// supplied actor, so the site query and deploy commands can be observed.
|
||||
/// supplied actor, so the site query and deploy notifies can be observed.
|
||||
/// <paramref name="centralFetchBaseUrl"/> seeds the DeploymentService's own
|
||||
/// <see cref="CommunicationOptions.CentralFetchBaseUrl"/>; pass "" to drive
|
||||
/// the notify-and-fetch fail-fast guard.
|
||||
/// </summary>
|
||||
private DeploymentService CreateServiceWithCommActor(IActorRef commActor)
|
||||
private DeploymentService CreateServiceWithCommActor(
|
||||
IActorRef commActor, string centralFetchBaseUrl = TestFetchBaseUrl)
|
||||
{
|
||||
var comms = new CommunicationService(
|
||||
Options.Create(new CommunicationOptions
|
||||
@@ -944,6 +1032,7 @@ public class DeploymentServiceTests : TestKit
|
||||
new RevisionHashService(),
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5) }),
|
||||
DeployCommOptions(centralFetchBaseUrl),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
}
|
||||
|
||||
@@ -1159,6 +1248,127 @@ public class DeploymentServiceTests : TestKit
|
||||
Assert.Equal(1, counters.DeployCount);
|
||||
}
|
||||
|
||||
// ── Task 6: notify-and-fetch send path (stage PendingDeployment + send RefreshDeploymentCommand) ──
|
||||
|
||||
[Fact]
|
||||
public async Task DeployInstanceAsync_Success_StagesPendingConfigThenSendsRefreshNotify()
|
||||
{
|
||||
// The central send path must stage the flattened config in a
|
||||
// PendingDeployment row and then notify the site with a small
|
||||
// RefreshDeploymentCommand (NOT ship the fat config inline), so the site
|
||||
// fetches the config over HTTP. The staged token + fetch base URL +
|
||||
// deployment id must travel in the notify.
|
||||
var instance = new Instance("NotifyFetchInst")
|
||||
{
|
||||
Id = 80, SiteId = 1, State = InstanceState.NotDeployed
|
||||
};
|
||||
_repo.GetInstanceByIdAsync(80, Arg.Any<CancellationToken>()).Returns(instance);
|
||||
SetupValidPipeline(80, "NotifyFetchInst", "sha256:target");
|
||||
_repo.GetCurrentDeploymentStatusAsync(80, Arg.Any<CancellationToken>())
|
||||
.Returns((DeploymentRecord?)null);
|
||||
|
||||
PendingDeployment? staged = null;
|
||||
await _repo.AddPendingDeploymentAsync(
|
||||
Arg.Do<PendingDeployment>(p => staged = p), Arg.Any<CancellationToken>());
|
||||
|
||||
var counters = new ReconcileProbeCounters();
|
||||
var commActor = Sys.ActorOf(Props.Create(() =>
|
||||
new ReconcileProbeActor(counters, siteHash: "sha256:target", failQuery: false)));
|
||||
var service = CreateServiceWithCommActor(commActor);
|
||||
|
||||
var result = await service.DeployInstanceAsync(80, "admin");
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
|
||||
// A PendingDeployment was staged with the deploy's identity + config.
|
||||
Assert.NotNull(staged);
|
||||
Assert.Equal(80, staged!.InstanceId);
|
||||
Assert.Equal("sha256:target", staged.RevisionHash);
|
||||
Assert.False(string.IsNullOrEmpty(staged.DeploymentId));
|
||||
Assert.False(string.IsNullOrEmpty(staged.Token));
|
||||
Assert.False(string.IsNullOrEmpty(staged.ConfigurationJson));
|
||||
// The TTL window is createdAt + CommunicationOptions.PendingDeploymentTtl.
|
||||
Assert.Equal(TestPendingTtl, staged.ExpiresAtUtc - staged.CreatedAtUtc);
|
||||
|
||||
// The site was notified via RefreshDeploymentCommand (the deploy path no
|
||||
// longer ships DeployInstanceCommand), carrying the SAME deployment id,
|
||||
// the configured fetch base URL, and the staged fetch token.
|
||||
Assert.Equal(1, counters.DeployCount);
|
||||
Assert.NotNull(counters.LastRefresh);
|
||||
Assert.Equal(staged.DeploymentId, counters.LastRefresh!.DeploymentId);
|
||||
Assert.Equal("NotifyFetchInst", counters.LastRefresh.InstanceUniqueName);
|
||||
Assert.Equal("sha256:target", counters.LastRefresh.RevisionHash);
|
||||
Assert.Equal(TestFetchBaseUrl, counters.LastRefresh.CentralFetchBaseUrl);
|
||||
Assert.Equal(staged.Token, counters.LastRefresh.FetchToken);
|
||||
|
||||
// The staged config was committed before/with the notify.
|
||||
await _repo.Received().AddPendingDeploymentAsync(
|
||||
Arg.Any<PendingDeployment>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeployInstanceAsync_Success_DoesNotDeletePendingRow()
|
||||
{
|
||||
// Cleanup is TTL-based only: deleting the pending row on success would
|
||||
// 404 a standby node whose HTTP fetch is still in flight after a
|
||||
// central-side Ask timeout. The row must be left for TTL purge.
|
||||
var instance = new Instance("NoDeleteInst")
|
||||
{
|
||||
Id = 81, SiteId = 1, State = InstanceState.NotDeployed
|
||||
};
|
||||
_repo.GetInstanceByIdAsync(81, Arg.Any<CancellationToken>()).Returns(instance);
|
||||
SetupValidPipeline(81, "NoDeleteInst", "sha256:target");
|
||||
_repo.GetCurrentDeploymentStatusAsync(81, Arg.Any<CancellationToken>())
|
||||
.Returns((DeploymentRecord?)null);
|
||||
|
||||
var counters = new ReconcileProbeCounters();
|
||||
var commActor = Sys.ActorOf(Props.Create(() =>
|
||||
new ReconcileProbeActor(counters, siteHash: "sha256:target", failQuery: false)));
|
||||
var service = CreateServiceWithCommActor(commActor);
|
||||
|
||||
var result = await service.DeployInstanceAsync(81, "admin");
|
||||
|
||||
Assert.True(result.IsSuccess);
|
||||
// The pending row is NOT deleted on success — TTL purge owns cleanup.
|
||||
await _repo.DidNotReceive().DeletePendingDeploymentByIdAsync(
|
||||
Arg.Any<string>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeployInstanceAsync_CentralFetchBaseUrlEmpty_FailsFastWithoutNotifyOrStaging()
|
||||
{
|
||||
// Notify-and-fetch requires CentralFetchBaseUrl. With it empty the deploy
|
||||
// must fail fast with a clear configuration error BEFORE staging a
|
||||
// pending row, creating a deployment record, or notifying the site — so
|
||||
// the operator never sees a confusing downstream site-fetch error.
|
||||
var instance = new Instance("NoFetchUrlInst")
|
||||
{
|
||||
Id = 82, SiteId = 1, State = InstanceState.NotDeployed
|
||||
};
|
||||
_repo.GetInstanceByIdAsync(82, Arg.Any<CancellationToken>()).Returns(instance);
|
||||
SetupValidPipeline(82, "NoFetchUrlInst", "sha256:target");
|
||||
_repo.GetCurrentDeploymentStatusAsync(82, Arg.Any<CancellationToken>())
|
||||
.Returns((DeploymentRecord?)null);
|
||||
|
||||
var counters = new ReconcileProbeCounters();
|
||||
var commActor = Sys.ActorOf(Props.Create(() =>
|
||||
new ReconcileProbeActor(counters, siteHash: "sha256:target", failQuery: false)));
|
||||
var service = CreateServiceWithCommActor(commActor, centralFetchBaseUrl: "");
|
||||
|
||||
var result = await service.DeployInstanceAsync(82, "admin");
|
||||
|
||||
Assert.True(result.IsFailure);
|
||||
Assert.Contains("CentralFetchBaseUrl", result.Error);
|
||||
|
||||
// No site notify, no staged config, no deployment record.
|
||||
Assert.Equal(0, counters.DeployCount);
|
||||
Assert.Null(counters.LastRefresh);
|
||||
await _repo.DidNotReceive().AddPendingDeploymentAsync(
|
||||
Arg.Any<PendingDeployment>(), Arg.Any<CancellationToken>());
|
||||
await _repo.DidNotReceive().AddDeploymentRecordAsync(
|
||||
Arg.Any<DeploymentRecord>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
// ── DeploymentManager-015: reconciliation must perform the normal success side effects ──
|
||||
|
||||
[Fact]
|
||||
@@ -1386,6 +1596,7 @@ public class DeploymentServiceTests : TestKit
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
LifecycleCommandTimeout = TimeSpan.FromMilliseconds(300)
|
||||
}),
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
@@ -1443,6 +1654,7 @@ public class DeploymentServiceTests : TestKit
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
LifecycleCommandTimeout = deadline,
|
||||
}),
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
|
||||
var result = await service.DisableInstanceAsync(61, "operator-jane");
|
||||
@@ -1492,6 +1704,7 @@ public class DeploymentServiceTests : TestKit
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
LifecycleCommandTimeout = TimeSpan.FromMilliseconds(300),
|
||||
}),
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
|
||||
var result = await service.EnableInstanceAsync(62, "operator-jane");
|
||||
@@ -1538,6 +1751,7 @@ public class DeploymentServiceTests : TestKit
|
||||
OperationLockTimeout = TimeSpan.FromSeconds(5),
|
||||
LifecycleCommandTimeout = TimeSpan.FromMilliseconds(300),
|
||||
}),
|
||||
DeployCommOptions(),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
|
||||
var result = await service.DeleteInstanceAsync(63, "operator-jane");
|
||||
@@ -1616,6 +1830,9 @@ public class DeploymentServiceTests : TestKit
|
||||
/// defers each deploy reply via the scheduler, so if two deploys for the
|
||||
/// same instance were NOT serialized by the operation lock their windows
|
||||
/// would overlap and <c>MaxConcurrent</c> would exceed 1.
|
||||
///
|
||||
/// Notify-and-fetch (Task 6): the deploy path now sends a
|
||||
/// <see cref="RefreshDeploymentCommand"/> instead of <c>DeployInstanceCommand</c>.
|
||||
/// </summary>
|
||||
private class SerializationProbeActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
@@ -1625,7 +1842,7 @@ public class DeploymentServiceTests : TestKit
|
||||
{
|
||||
Receive<SiteEnvelope>(env =>
|
||||
{
|
||||
if (env.Message is DeployInstanceCommand d)
|
||||
if (env.Message is RefreshDeploymentCommand d)
|
||||
{
|
||||
lock (counters.Gate)
|
||||
{
|
||||
@@ -1661,7 +1878,7 @@ public class DeploymentServiceTests : TestKit
|
||||
});
|
||||
}
|
||||
|
||||
private sealed record CompleteDeploy(DeployInstanceCommand Command, IActorRef ReplyTo);
|
||||
private sealed record CompleteDeploy(RefreshDeploymentCommand Command, IActorRef ReplyTo);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -1673,14 +1890,27 @@ public class DeploymentServiceTests : TestKit
|
||||
{
|
||||
public int QueryCount;
|
||||
public int DeployCount;
|
||||
|
||||
/// <summary>
|
||||
/// Notify-and-fetch (Task 6): the most recent
|
||||
/// <see cref="RefreshDeploymentCommand"/> the probe received, so a test
|
||||
/// can assert the deploy path notified the site with the staged token,
|
||||
/// fetch base URL, and deployment id.
|
||||
/// </summary>
|
||||
public RefreshDeploymentCommand? LastRefresh;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stand-in CentralCommunicationActor for reconciliation tests. Counts the
|
||||
/// site queries and deploy commands it receives (into a per-test
|
||||
/// site queries and deploy notifies it receives (into a per-test
|
||||
/// <see cref="ReconcileProbeCounters"/> instance), answers queries with a
|
||||
/// configurable applied revision hash, and (optionally) drops the query to
|
||||
/// simulate an unreachable site so the central Ask times out.
|
||||
///
|
||||
/// Notify-and-fetch (Task 6): the deploy path now sends a
|
||||
/// <see cref="RefreshDeploymentCommand"/> (not the fat
|
||||
/// <c>DeployInstanceCommand</c>), so the probe counts that message as a
|
||||
/// "deploy" and records it for assertion.
|
||||
/// </summary>
|
||||
private class ReconcileProbeActor : ReceiveActor
|
||||
{
|
||||
@@ -1701,8 +1931,9 @@ public class DeploymentServiceTests : TestKit
|
||||
// failQuery: drop the message -> caller's Ask times out.
|
||||
break;
|
||||
|
||||
case DeployInstanceCommand d:
|
||||
case RefreshDeploymentCommand d:
|
||||
Interlocked.Increment(ref counters.DeployCount);
|
||||
counters.LastRefresh = d;
|
||||
Sender.Tell(new DeploymentStatusResponse(
|
||||
d.DeploymentId, d.InstanceUniqueName,
|
||||
DeploymentStatus.Success, null, DateTimeOffset.UtcNow));
|
||||
@@ -1726,4 +1957,13 @@ public class DeploymentServiceTests : TestKit
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Akka actor that silently discards every message, causing any Ask call
|
||||
/// directed at it to time out with <see cref="Akka.Actor.AskTimeoutException"/>.
|
||||
/// Used by <c>DeployInstanceAsync_AskTimeoutException_ClassifiedAsTimeout</c>
|
||||
/// to verify that the deploy catch block treats
|
||||
/// <see cref="Akka.Actor.AskTimeoutException"/> as a timeout.
|
||||
/// </summary>
|
||||
private class DroppingActor : ReceiveActor { }
|
||||
}
|
||||
|
||||
@@ -61,6 +61,10 @@ public class DeploymentStatusNotifierTests : TestKit
|
||||
_service = new DeploymentService(
|
||||
_repo, siteRepo, _pipeline, _comms, _lockManager, _audit,
|
||||
new DiffService(), new RevisionHashService(), _notifier, options,
|
||||
// Notify-and-fetch (Task 6): a non-empty CentralFetchBaseUrl lets the
|
||||
// deploy path reach the send (and, here, the catch block) rather than
|
||||
// tripping the fail-fast configuration guard.
|
||||
Options.Create(new CommunicationOptions { CentralFetchBaseUrl = "https://central.test:9000" }),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ManagementService;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration coverage for the "notify-and-fetch" deploy seam: central stages a flattened
|
||||
/// config in a <see cref="PendingDeployment"/> row and serves it from
|
||||
/// <c>GET /api/internal/deployments/{deploymentId}/config</c> (token in the
|
||||
/// <c>X-Deployment-Token</c> header); the SITE fetches it over HTTP via
|
||||
/// <see cref="HttpDeploymentConfigFetcher"/>. This decouples config size from Akka's 128 KB
|
||||
/// remote-frame limit.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Harness.</b> A minimal in-memory ASP.NET Core <see cref="TestServer"/> (NOT the full
|
||||
/// <c>WebApplicationFactory<Program></c>) maps ONLY the real
|
||||
/// <see cref="DeploymentConfigEndpoints.MapDeploymentConfigAPI"/> endpoint over a real
|
||||
/// <see cref="DeploymentManagerRepository"/> backed by a real (SQLite in-memory)
|
||||
/// <see cref="ScadaBridgeDbContext"/>. The light TestServer is preferred over the full Host
|
||||
/// because the central Host startup (migrations, Akka remoting, cluster bootstrap) cannot run
|
||||
/// in this sandbox — and the real seam under test is the production serve-endpoint plus the
|
||||
/// production fetcher over an in-memory HTTP transport, both of which run hermetically.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Sandbox note.</b> A full two-cluster end-to-end over Akka.Remote dot-netty sockets is
|
||||
/// NOT exercised here (sockets cannot bind in this sandbox); the actor-level apply/replication
|
||||
/// is covered by unit tests and the real cluster by a separate live-smoke task. These tests
|
||||
/// prove the HTTP seam — the part the size-limit fix hinges on.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class DeployNotifyFetchTests
|
||||
{
|
||||
/// <summary>
|
||||
/// SQLite-adapted <see cref="ScadaBridgeDbContext"/>: maps <see cref="DateTimeOffset"/> columns
|
||||
/// to sortable ISO 8601 strings (SQLite has no native DateTimeOffset), drops the SQL Server
|
||||
/// rowversion concurrency token on <see cref="DeploymentRecord"/>, and is constructed with an
|
||||
/// ephemeral Data Protection provider so secret-bearing columns are write-capable.
|
||||
/// </summary>
|
||||
private sealed class SqliteFetchDbContext : ScadaBridgeDbContext
|
||||
{
|
||||
public SqliteFetchDbContext(DbContextOptions<ScadaBridgeDbContext> options)
|
||||
: base(options, new EphemeralDataProtectionProvider())
|
||||
{
|
||||
}
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
{
|
||||
base.OnModelCreating(modelBuilder);
|
||||
|
||||
// SQLite cannot auto-generate SQL Server rowversion values.
|
||||
modelBuilder.Entity<DeploymentRecord>(builder =>
|
||||
{
|
||||
builder.Property(d => d.RowVersion)
|
||||
.IsRequired(false)
|
||||
.IsConcurrencyToken(false)
|
||||
.ValueGeneratedNever();
|
||||
});
|
||||
|
||||
var converter = new ValueConverter<DateTimeOffset, string>(
|
||||
v => v.UtcDateTime.ToString("o"),
|
||||
v => DateTimeOffset.Parse(v));
|
||||
var nullableConverter = new ValueConverter<DateTimeOffset?, string?>(
|
||||
v => v.HasValue ? v.Value.UtcDateTime.ToString("o") : null,
|
||||
v => v != null ? DateTimeOffset.Parse(v) : null);
|
||||
|
||||
foreach (var entityType in modelBuilder.Model.GetEntityTypes())
|
||||
{
|
||||
foreach (var property in entityType.GetProperties())
|
||||
{
|
||||
if (property.ClrType == typeof(DateTimeOffset))
|
||||
{
|
||||
property.SetValueConverter(converter);
|
||||
property.SetColumnType("TEXT");
|
||||
}
|
||||
else if (property.ClrType == typeof(DateTimeOffset?))
|
||||
{
|
||||
property.SetValueConverter(nullableConverter);
|
||||
property.SetColumnType("TEXT");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One self-contained test fixture: a kept-open SQLite in-memory connection (an in-memory
|
||||
/// database is destroyed when its last connection closes), an in-memory <see cref="TestServer"/>
|
||||
/// hosting ONLY the real config-fetch endpoint over a real repository, and the in-memory
|
||||
/// <see cref="HttpClient"/> the production fetcher drives.
|
||||
/// </summary>
|
||||
private sealed class FetchHarness : IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _connection;
|
||||
private readonly IHost _host;
|
||||
private readonly DbContextOptions<ScadaBridgeDbContext> _dbOptions;
|
||||
|
||||
public HttpClient Client { get; }
|
||||
|
||||
/// <summary>Base URL the site presents to the fetcher (the test server's address).</summary>
|
||||
public string BaseUrl { get; }
|
||||
|
||||
private FetchHarness(
|
||||
SqliteConnection connection,
|
||||
IHost host,
|
||||
DbContextOptions<ScadaBridgeDbContext> dbOptions,
|
||||
HttpClient client)
|
||||
{
|
||||
_connection = connection;
|
||||
_host = host;
|
||||
_dbOptions = dbOptions;
|
||||
Client = client;
|
||||
BaseUrl = client.BaseAddress!.ToString();
|
||||
}
|
||||
|
||||
public static async Task<FetchHarness> CreateAsync()
|
||||
{
|
||||
var connection = new SqliteConnection("DataSource=:memory:");
|
||||
connection.Open();
|
||||
|
||||
var dbOptions = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlite(connection)
|
||||
.ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning))
|
||||
.Options;
|
||||
|
||||
// Create the schema once on a throwaway context over the shared connection.
|
||||
using (var schemaContext = new SqliteFetchDbContext(dbOptions))
|
||||
{
|
||||
await schemaContext.Database.EnsureCreatedAsync();
|
||||
}
|
||||
|
||||
// Minimal host: routing + the REAL serve-endpoint over the REAL repository.
|
||||
var host = await new HostBuilder()
|
||||
.ConfigureWebHost(webBuilder =>
|
||||
{
|
||||
webBuilder
|
||||
.UseTestServer()
|
||||
.ConfigureServices(services =>
|
||||
{
|
||||
services.AddRouting();
|
||||
services.AddScoped<ScadaBridgeDbContext>(_ => new SqliteFetchDbContext(dbOptions));
|
||||
services.AddScoped<IDeploymentManagerRepository>(sp =>
|
||||
new DeploymentManagerRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
})
|
||||
.Configure(app =>
|
||||
{
|
||||
app.UseRouting();
|
||||
app.UseEndpoints(endpoints => endpoints.MapDeploymentConfigAPI());
|
||||
});
|
||||
})
|
||||
.StartAsync();
|
||||
|
||||
var client = host.GetTestClient();
|
||||
return new FetchHarness(connection, host, dbOptions, client);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Seeds an <see cref="Instance"/> (with its required <see cref="Site"/> + <see cref="Template"/>)
|
||||
/// and returns its generated id. The PendingDeployment → Instance FK is enforced by SQLite, so a
|
||||
/// real row is required.
|
||||
/// </summary>
|
||||
public async Task<int> SeedInstanceAsync(string uniqueName)
|
||||
{
|
||||
await using var ctx = new SqliteFetchDbContext(_dbOptions);
|
||||
var site = new Site($"Site-{uniqueName}", $"S-{uniqueName}");
|
||||
var template = new Template($"T-{uniqueName}");
|
||||
ctx.Sites.Add(site);
|
||||
ctx.Templates.Add(template);
|
||||
await ctx.SaveChangesAsync();
|
||||
|
||||
var instance = new Instance(uniqueName) { SiteId = site.Id, TemplateId = template.Id };
|
||||
ctx.Instances.Add(instance);
|
||||
await ctx.SaveChangesAsync();
|
||||
return instance.Id;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stages a pending deployment through the REAL repository path
|
||||
/// (<see cref="DeploymentManagerRepository.AddPendingDeploymentAsync"/> + SaveChanges), so
|
||||
/// supersession of any prior row for the same instance is exercised exactly as in production.
|
||||
/// </summary>
|
||||
public async Task StageAsync(PendingDeployment pending)
|
||||
{
|
||||
await using var ctx = new SqliteFetchDbContext(_dbOptions);
|
||||
var repo = new DeploymentManagerRepository(ctx);
|
||||
await repo.AddPendingDeploymentAsync(pending);
|
||||
await repo.SaveChangesAsync();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Client.Dispose();
|
||||
_host.Dispose();
|
||||
_connection.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private static HttpDeploymentConfigFetcher NewFetcher(FetchHarness harness)
|
||||
=> new(harness.Client, NullLogger<HttpDeploymentConfigFetcher>.Instance);
|
||||
|
||||
private static PendingDeployment NewPending(
|
||||
string deploymentId, int instanceId, string config, string token, DateTimeOffset expiresAtUtc)
|
||||
=> new(
|
||||
deploymentId, instanceId, revisionHash: "rev-" + deploymentId,
|
||||
configurationJson: config, token: token,
|
||||
createdAtUtc: DateTimeOffset.UtcNow.AddMinutes(-1), expiresAtUtc: expiresAtUtc);
|
||||
|
||||
/// <summary>
|
||||
/// Deterministic JSON string comfortably larger than Akka's 128 KB remote-frame limit
|
||||
/// (~204 KB here). It need not be a real FlattenedConfiguration — the endpoint serves the
|
||||
/// stored string verbatim.
|
||||
/// </summary>
|
||||
private static string BuildLargeConfigJson()
|
||||
{
|
||||
// 200 KiB of payload + JSON wrapper. ASCII only, so byte-for-byte == char-for-char.
|
||||
var payload = new string('A', 200 * 1024);
|
||||
return "{\"payload\":\"" + payload + "\"}";
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
// 1. Large config round-trips: the core proof that the 128 KB size limit is gone.
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_LargeConfigOver128Kb_RoundTripsExactBytes()
|
||||
{
|
||||
using var harness = await FetchHarness.CreateAsync();
|
||||
var fetcher = NewFetcher(harness);
|
||||
var instanceId = await harness.SeedInstanceAsync("LargeInst");
|
||||
|
||||
var largeJson = BuildLargeConfigJson();
|
||||
Assert.True(
|
||||
largeJson.Length > 128 * 1024,
|
||||
$"staged config must exceed the old 128 KB Akka frame limit; was {largeJson.Length} bytes");
|
||||
|
||||
await harness.StageAsync(NewPending(
|
||||
"dep-large", instanceId, largeJson, token: "tok-large",
|
||||
expiresAtUtc: DateTimeOffset.UtcNow.AddMinutes(10)));
|
||||
|
||||
var fetched = await fetcher.FetchAsync(harness.BaseUrl, "dep-large", "tok-large", CancellationToken.None);
|
||||
|
||||
// Byte-for-byte (ordinal) round-trip of a >128 KB body over the HTTP seam.
|
||||
Assert.Equal(largeJson, fetched, StringComparer.Ordinal);
|
||||
Assert.True(
|
||||
fetched.Length > 128 * 1024,
|
||||
$"fetched config must exceed the old 128 KB Akka frame limit; was {fetched.Length} bytes");
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
// 2. Supersession: staging B for an instance deletes its prior pending row A.
|
||||
// Fetching A -> 404 (IsSuperseded); fetching B -> B's config.
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_SupersededDeployment_Throws404SupersededAndServesNewest()
|
||||
{
|
||||
using var harness = await FetchHarness.CreateAsync();
|
||||
var fetcher = NewFetcher(harness);
|
||||
var instanceId = await harness.SeedInstanceAsync("SupersedeInst");
|
||||
var future = DateTimeOffset.UtcNow.AddMinutes(10);
|
||||
|
||||
await harness.StageAsync(NewPending("dep-A", instanceId, "{\"v\":1}", "tok-A", future));
|
||||
// Staging B for the SAME instance supersedes (deletes) A via the real repository path.
|
||||
await harness.StageAsync(NewPending("dep-B", instanceId, "{\"v\":2}", "tok-B", future));
|
||||
|
||||
// A is gone -> 404 -> IsSuperseded.
|
||||
var ex = await Assert.ThrowsAsync<DeploymentConfigFetchException>(
|
||||
() => fetcher.FetchAsync(harness.BaseUrl, "dep-A", "tok-A", CancellationToken.None));
|
||||
Assert.True(ex.IsSuperseded);
|
||||
|
||||
// B is current and still served.
|
||||
var fetchedB = await fetcher.FetchAsync(harness.BaseUrl, "dep-B", "tok-B", CancellationToken.None);
|
||||
Assert.Equal("{\"v\":2}", fetchedB, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
// 3. Wrong token: a live, non-expired row with a bad token -> 401 (NOT 404).
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_WrongToken_Throws401NotSuperseded()
|
||||
{
|
||||
using var harness = await FetchHarness.CreateAsync();
|
||||
var fetcher = NewFetcher(harness);
|
||||
var instanceId = await harness.SeedInstanceAsync("TokenInst");
|
||||
|
||||
await harness.StageAsync(NewPending(
|
||||
"dep-tok", instanceId, "{\"v\":1}", token: "right-token",
|
||||
expiresAtUtc: DateTimeOffset.UtcNow.AddMinutes(10)));
|
||||
|
||||
var ex = await Assert.ThrowsAsync<DeploymentConfigFetchException>(
|
||||
() => fetcher.FetchAsync(harness.BaseUrl, "dep-tok", "wrong-token", CancellationToken.None));
|
||||
|
||||
// 401 (not 404): a valid, live id with a bad token is unauthorized, not "gone".
|
||||
Assert.False(ex.IsSuperseded);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
// 4. Expired: a row past its TTL is indistinguishable from gone -> 404 (IsSuperseded),
|
||||
// even with the correct token.
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_ExpiredRow_Throws404SupersededEvenWithCorrectToken()
|
||||
{
|
||||
using var harness = await FetchHarness.CreateAsync();
|
||||
var fetcher = NewFetcher(harness);
|
||||
var instanceId = await harness.SeedInstanceAsync("ExpiredInst");
|
||||
|
||||
await harness.StageAsync(NewPending(
|
||||
"dep-exp", instanceId, "{\"v\":1}", token: "tok-exp",
|
||||
expiresAtUtc: DateTimeOffset.UtcNow.AddMinutes(-1))); // TTL already elapsed
|
||||
|
||||
var ex = await Assert.ThrowsAsync<DeploymentConfigFetchException>(
|
||||
() => fetcher.FetchAsync(harness.BaseUrl, "dep-exp", "tok-exp", CancellationToken.None));
|
||||
|
||||
// Expired is hidden as 404 (existence-hiding), so IsSuperseded is set.
|
||||
Assert.True(ex.IsSuperseded);
|
||||
}
|
||||
}
|
||||
+3
@@ -16,6 +16,7 @@
|
||||
<PackageReference Include="Akka.TestKit.Xunit2" />
|
||||
<PackageReference Include="coverlet.collector" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.TestHost" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
@@ -36,6 +37,8 @@
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Host/ZB.MOM.WW.ScadaBridge.Host.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.NotificationOutbox/ZB.MOM.WW.ScadaBridge.NotificationOutbox.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.ManagementService/ZB.MOM.WW.ScadaBridge.ManagementService.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Http.HttpResults;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.ManagementService;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ManagementService.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for the pure decision resolver behind the token-gated internal
|
||||
/// deployment-config fetch endpoint (<see cref="DeploymentConfigEndpoints.Resolve"/>).
|
||||
///
|
||||
/// <para>
|
||||
/// This endpoint is machine-to-machine: a site presents a per-deployment fetch
|
||||
/// token in the <c>X-Deployment-Token</c> header and the token is the entire
|
||||
/// security boundary. The resolver is extracted so the security-critical
|
||||
/// decision logic is unit-testable without standing up a web host. The mapping
|
||||
/// from the resolver result to the documented HTTP status codes is:
|
||||
/// <list type="bullet">
|
||||
/// <item>unknown / superseded (null row) → 404 NotFound</item>
|
||||
/// <item>expired row → 404 NotFound (does not leak existence to a wrong token)</item>
|
||||
/// <item>missing / wrong token → 401 Unauthorized</item>
|
||||
/// <item>valid row + correct token + not expired → 200 with the staged JSON</item>
|
||||
/// </list>
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class DeploymentConfigEndpointsTests
|
||||
{
|
||||
private const string ValidToken = "test-deployment-token-abc123";
|
||||
private const string ConfigJson = "{\"instanceId\":42,\"attributes\":[]}";
|
||||
|
||||
private static readonly DateTimeOffset Now = new(2026, 6, 26, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
private static PendingDeployment Row(string token = ValidToken, DateTimeOffset? expiresAtUtc = null) =>
|
||||
new(
|
||||
deploymentId: "deploy-1",
|
||||
instanceId: 42,
|
||||
revisionHash: "hash-1",
|
||||
configurationJson: ConfigJson,
|
||||
token: token,
|
||||
createdAtUtc: Now.AddMinutes(-1),
|
||||
expiresAtUtc: expiresAtUtc ?? Now.AddMinutes(5));
|
||||
|
||||
private static int? StatusOf(IResult result) => (result as IStatusCodeHttpResult)?.StatusCode;
|
||||
|
||||
[Fact]
|
||||
public void Resolve_NullRow_ReturnsNotFound()
|
||||
{
|
||||
var result = DeploymentConfigEndpoints.Resolve(row: null, token: ValidToken, nowUtc: Now);
|
||||
|
||||
Assert.IsType<NotFound>(result);
|
||||
Assert.Equal(StatusCodes.Status404NotFound, StatusOf(result));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_ExpiredRow_WithCorrectToken_ReturnsNotFound()
|
||||
{
|
||||
// Expired exactly at now: ExpiresAtUtc <= now is the boundary that purges.
|
||||
var row = Row(expiresAtUtc: Now);
|
||||
|
||||
var result = DeploymentConfigEndpoints.Resolve(row, token: ValidToken, nowUtc: Now);
|
||||
|
||||
Assert.IsType<NotFound>(result);
|
||||
Assert.Equal(StatusCodes.Status404NotFound, StatusOf(result));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_ExpiredRowInThePast_WithCorrectToken_ReturnsNotFound()
|
||||
{
|
||||
var row = Row(expiresAtUtc: Now.AddSeconds(-1));
|
||||
|
||||
var result = DeploymentConfigEndpoints.Resolve(row, token: ValidToken, nowUtc: Now);
|
||||
|
||||
Assert.IsType<NotFound>(result);
|
||||
Assert.Equal(StatusCodes.Status404NotFound, StatusOf(result));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_ValidRow_WrongToken_ReturnsUnauthorized()
|
||||
{
|
||||
var row = Row();
|
||||
|
||||
var result = DeploymentConfigEndpoints.Resolve(row, token: "wrong-token", nowUtc: Now);
|
||||
|
||||
Assert.IsType<UnauthorizedHttpResult>(result);
|
||||
Assert.Equal(StatusCodes.Status401Unauthorized, StatusOf(result));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_ValidRow_EmptyToken_ReturnsUnauthorized()
|
||||
{
|
||||
var row = Row();
|
||||
|
||||
var result = DeploymentConfigEndpoints.Resolve(row, token: string.Empty, nowUtc: Now);
|
||||
|
||||
Assert.IsType<UnauthorizedHttpResult>(result);
|
||||
Assert.Equal(StatusCodes.Status401Unauthorized, StatusOf(result));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_ValidRow_NullToken_ReturnsUnauthorized()
|
||||
{
|
||||
var row = Row();
|
||||
|
||||
var result = DeploymentConfigEndpoints.Resolve(row, token: null!, nowUtc: Now);
|
||||
|
||||
Assert.IsType<UnauthorizedHttpResult>(result);
|
||||
Assert.Equal(StatusCodes.Status401Unauthorized, StatusOf(result));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_ValidRow_CorrectToken_NotExpired_ReturnsConfigJson()
|
||||
{
|
||||
var row = Row();
|
||||
|
||||
var result = DeploymentConfigEndpoints.Resolve(row, token: ValidToken, nowUtc: Now);
|
||||
|
||||
var content = Assert.IsType<ContentHttpResult>(result);
|
||||
Assert.Equal(ConfigJson, content.ResponseContent);
|
||||
Assert.Equal("application/json", content.ContentType);
|
||||
}
|
||||
}
|
||||
+133
-2
@@ -10,6 +10,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
@@ -48,7 +49,8 @@ public class DeploymentManagerActorTests : TestKit, IDisposable
|
||||
}
|
||||
|
||||
private IActorRef CreateDeploymentManager(
|
||||
SiteRuntimeOptions? options = null, IServiceProvider? serviceProvider = null)
|
||||
SiteRuntimeOptions? options = null, IServiceProvider? serviceProvider = null,
|
||||
IDeploymentConfigFetcher? configFetcher = null)
|
||||
{
|
||||
options ??= new SiteRuntimeOptions();
|
||||
return ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
@@ -62,7 +64,8 @@ public class DeploymentManagerActorTests : TestKit, IDisposable
|
||||
null,
|
||||
null,
|
||||
serviceProvider,
|
||||
null)));
|
||||
null,
|
||||
configFetcher)));
|
||||
}
|
||||
|
||||
private static string MakeConfigJson(string instanceName)
|
||||
@@ -675,4 +678,132 @@ public class DeploymentManagerActorTests : TestKit, IDisposable
|
||||
Assert.False(snapshot.InstanceNotFound,
|
||||
"A deployed (but empty) instance must NOT set InstanceNotFound.");
|
||||
}
|
||||
|
||||
// ── Task 10: notify-and-fetch — RefreshDeploymentCommand → fetch → apply ──
|
||||
|
||||
[Fact]
|
||||
public async Task RefreshDeployment_FetchSucceeds_AppliesConfigAndRepliesSuccess()
|
||||
{
|
||||
// The active singleton receives a small RefreshDeploymentCommand, fetches the
|
||||
// flattened config over HTTP via IDeploymentConfigFetcher, then runs the existing
|
||||
// apply path: the config is persisted to SQLite and a Success status is replied.
|
||||
var fetcher = new FakeConfigFetcher(MakeConfigJson("FetchedPump"));
|
||||
var actor = CreateDeploymentManager(configFetcher: fetcher);
|
||||
await Task.Delay(500); // empty startup
|
||||
|
||||
actor.Tell(new RefreshDeploymentCommand(
|
||||
"dep-fetch-1", "FetchedPump", "sha256:fetch", "admin", DateTimeOffset.UtcNow,
|
||||
"http://central:9000", "tok-123"));
|
||||
|
||||
var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(DeploymentStatus.Success, response.Status);
|
||||
Assert.Equal("FetchedPump", response.InstanceUniqueName);
|
||||
Assert.Equal("dep-fetch-1", response.DeploymentId);
|
||||
|
||||
// The fetcher was called with the command's fetch coordinates.
|
||||
Assert.Equal("http://central:9000", fetcher.LastBaseUrl);
|
||||
Assert.Equal("dep-fetch-1", fetcher.LastDeploymentId);
|
||||
Assert.Equal("tok-123", fetcher.LastToken);
|
||||
|
||||
// The existing apply path ran end-to-end: the fetched config is persisted.
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.Contains(configs, c => c.InstanceUniqueName == "FetchedPump");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RefreshDeployment_FetchFails_RepliesFailedAndDoesNotApply()
|
||||
{
|
||||
// A config fetch failure (DeploymentConfigFetchException surfaces as a faulted
|
||||
// Task) must reply Failed with a "Communication failure:" message and must NOT
|
||||
// apply anything — no config persisted, no Instance Actor created.
|
||||
var fetcher = new FakeConfigFetcher(
|
||||
new DeploymentConfigFetchException("central unreachable", isSuperseded: false));
|
||||
var actor = CreateDeploymentManager(configFetcher: fetcher);
|
||||
await Task.Delay(500);
|
||||
|
||||
actor.Tell(new RefreshDeploymentCommand(
|
||||
"dep-fetch-fail", "UnfetchedPump", "sha256:x", "admin", DateTimeOffset.UtcNow,
|
||||
"http://central:9000", "tok-x"));
|
||||
|
||||
var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(DeploymentStatus.Failed, response.Status);
|
||||
Assert.Equal("UnfetchedPump", response.InstanceUniqueName);
|
||||
Assert.NotNull(response.ErrorMessage);
|
||||
Assert.StartsWith("Communication failure:", response.ErrorMessage!);
|
||||
|
||||
// No apply occurred: nothing was persisted for the instance.
|
||||
await Task.Delay(500);
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "UnfetchedPump");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RefreshDeployment_NullFetcher_RepliesFailed()
|
||||
{
|
||||
// A node with no configured fetcher (the `_configFetcher is null` guard) must reply
|
||||
// Failed rather than NRE-crashing or silently dropping the central Ask.
|
||||
var actor = CreateDeploymentManager(); // no configFetcher
|
||||
|
||||
actor.Tell(new RefreshDeploymentCommand(
|
||||
"dep-null", "Pump", "sha256:x", "admin", DateTimeOffset.UtcNow,
|
||||
"http://central:9000", "tok"));
|
||||
|
||||
var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(2));
|
||||
Assert.Equal(DeploymentStatus.Failed, response.Status);
|
||||
Assert.Equal("Pump", response.InstanceUniqueName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RefreshDeployment_ReplyGoesToOriginalSender_AcrossAsyncFetch()
|
||||
{
|
||||
// The reply must reach the ORIGINAL sender (central's Ask temp actor), proving
|
||||
// the captured replyTo survives the async fetch + PipeTo continuation — where
|
||||
// Akka's Sender is no longer valid. Send with an explicit probe as the sender
|
||||
// and assert the probe (not the default test actor) receives the response.
|
||||
var fetcher = new FakeConfigFetcher(MakeConfigJson("SenderPump"));
|
||||
var probe = CreateTestProbe();
|
||||
var actor = CreateDeploymentManager(configFetcher: fetcher);
|
||||
await Task.Delay(500);
|
||||
|
||||
actor.Tell(new RefreshDeploymentCommand(
|
||||
"dep-sender", "SenderPump", "sha256:s", "admin", DateTimeOffset.UtcNow,
|
||||
"http://central:9000", "tok-s"), probe.Ref);
|
||||
|
||||
var response = probe.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(DeploymentStatus.Success, response.Status);
|
||||
Assert.Equal("SenderPump", response.InstanceUniqueName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-test fake <see cref="IDeploymentConfigFetcher"/>: returns a canned config JSON
|
||||
/// (notify-and-fetch success) or throws a canned exception (fetch failure), and records
|
||||
/// the fetch coordinates it was called with. It is async so a throw surfaces as a faulted
|
||||
/// <see cref="Task"/> — mirroring the real HttpDeploymentConfigFetcher; a synchronous
|
||||
/// throw would instead crash the actor before the ContinueWith/PipeTo could produce a
|
||||
/// RefreshFetchFailed.
|
||||
/// </summary>
|
||||
private sealed class FakeConfigFetcher : IDeploymentConfigFetcher
|
||||
{
|
||||
private readonly string? _result;
|
||||
private readonly Exception? _error;
|
||||
|
||||
public FakeConfigFetcher(string result) => _result = result;
|
||||
public FakeConfigFetcher(Exception error) => _error = error;
|
||||
|
||||
public string? LastBaseUrl { get; private set; }
|
||||
public string? LastDeploymentId { get; private set; }
|
||||
public string? LastToken { get; private set; }
|
||||
|
||||
public async Task<string> FetchAsync(
|
||||
string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct)
|
||||
{
|
||||
LastBaseUrl = centralFetchBaseUrl;
|
||||
LastDeploymentId = deploymentId;
|
||||
LastToken = token;
|
||||
await Task.Yield();
|
||||
if (_error != null)
|
||||
throw _error;
|
||||
return _result!;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="SiteReconciliationActor"/>: the per-node startup self-heal that
|
||||
/// reports the node's local deployed inventory to central (over the SiteCommunicationActor
|
||||
/// Ask), fetches the gap (missing/stale configs) over HTTP, guarded-writes them, and only
|
||||
/// LOGS orphans (never deletes). Best-effort throughout — a central-unreachable Ask or a
|
||||
/// per-item fetch failure must never crash the actor.
|
||||
/// </summary>
|
||||
public class SiteReconciliationActorTests : TestKit, IDisposable
|
||||
{
|
||||
private const string SiteIdentifier = "site-1";
|
||||
private const string NodeId = "node-a";
|
||||
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly string _dbFile;
|
||||
|
||||
public SiteReconciliationActorTests()
|
||||
{
|
||||
_dbFile = Path.Combine(Path.GetTempPath(), $"site-reconcile-test-{Guid.NewGuid():N}.db");
|
||||
_storage = new SiteStorageService(
|
||||
$"Data Source={_dbFile}", Microsoft.Extensions.Logging.Abstractions.NullLogger<SiteStorageService>.Instance);
|
||||
_storage.InitializeAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
Shutdown();
|
||||
try { File.Delete(_dbFile); } catch { /* cleanup */ }
|
||||
}
|
||||
|
||||
private IActorRef CreateReconciliationActor(
|
||||
IActorRef siteCommunicationActor,
|
||||
IDeploymentConfigFetcher fetcher,
|
||||
ILogger<SiteReconciliationActor>? logger = null,
|
||||
TimeSpan? askTimeout = null) =>
|
||||
ActorOf(Props.Create(() => new SiteReconciliationActor(
|
||||
_storage,
|
||||
fetcher,
|
||||
siteCommunicationActor,
|
||||
SiteIdentifier,
|
||||
NodeId,
|
||||
logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger<SiteReconciliationActor>.Instance,
|
||||
TimeSpan.FromMilliseconds(50), // initial delay — fast for tests
|
||||
askTimeout ?? TimeSpan.FromSeconds(5)))); // ask timeout
|
||||
|
||||
[Fact]
|
||||
public async Task MissingInstance_IsFetchedAndStored()
|
||||
{
|
||||
// Local inventory has A@rev1; central reports B is missing → fetch + store B.
|
||||
await _storage.StoreDeployedConfigAsync("A", "{\"instanceUniqueName\":\"A\"}", "depA", "rev1", true);
|
||||
|
||||
const string configB = "{\"instanceUniqueName\":\"B\"}";
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult(configB));
|
||||
var commProbe = CreateTestProbe();
|
||||
|
||||
CreateReconciliationActor(commProbe, fetcher);
|
||||
|
||||
var req = commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(SiteIdentifier, req.SiteIdentifier);
|
||||
Assert.Equal(NodeId, req.NodeId);
|
||||
Assert.True(req.LocalNameToRevisionHash.TryGetValue("A", out var hashA));
|
||||
Assert.Equal("rev1", hashA);
|
||||
|
||||
commProbe.Reply(new ReconcileSiteResponse(
|
||||
[new ReconcileGapItem("B", "depB", "rev2", true, "tok-b")],
|
||||
[],
|
||||
"http://central:9000"));
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
var row = Assert.Single(configs, c => c.InstanceUniqueName == "B");
|
||||
Assert.Equal(configB, row.ConfigJson);
|
||||
Assert.Equal("depB", row.DeploymentId);
|
||||
Assert.Equal("rev2", row.RevisionHash);
|
||||
Assert.True(row.IsEnabled);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
var call = Assert.Single(fetcher.Calls);
|
||||
Assert.Equal("http://central:9000", call.BaseUrl);
|
||||
Assert.Equal("depB", call.DeploymentId);
|
||||
Assert.Equal("tok-b", call.Token);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StaleInstance_IsRefreshed()
|
||||
{
|
||||
// Local A@rev1; central reports A is stale (now depA2/rev2) → fetch + guarded-write A.
|
||||
await _storage.StoreDeployedConfigAsync("A", "{\"old\":true}", "depA", "rev1", true);
|
||||
|
||||
const string configA2 = "{\"instanceUniqueName\":\"A\",\"v\":2}";
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult(configA2));
|
||||
var commProbe = CreateTestProbe();
|
||||
|
||||
CreateReconciliationActor(commProbe, fetcher);
|
||||
|
||||
commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
||||
commProbe.Reply(new ReconcileSiteResponse(
|
||||
[new ReconcileGapItem("A", "depA2", "rev2", false, "tok-a")],
|
||||
[],
|
||||
"http://central:9000"));
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
var row = Assert.Single(configs, c => c.InstanceUniqueName == "A");
|
||||
Assert.Equal(configA2, row.ConfigJson);
|
||||
Assert.Equal("depA2", row.DeploymentId);
|
||||
Assert.Equal("rev2", row.RevisionHash);
|
||||
Assert.False(row.IsEnabled);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
var call = Assert.Single(fetcher.Calls);
|
||||
Assert.Equal("depA2", call.DeploymentId);
|
||||
Assert.Equal("tok-a", call.Token);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Orphan_IsLoggedNotDeleted()
|
||||
{
|
||||
// Local Z exists; central reports Z as an orphan → log a warning, leave Z in place.
|
||||
await _storage.StoreDeployedConfigAsync("Z", "{\"instanceUniqueName\":\"Z\"}", "depZ", "revZ", true);
|
||||
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("never"));
|
||||
var logger = new CapturingLogger<SiteReconciliationActor>();
|
||||
var commProbe = CreateTestProbe();
|
||||
|
||||
CreateReconciliationActor(commProbe, fetcher, logger);
|
||||
|
||||
commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
||||
commProbe.Reply(new ReconcileSiteResponse([], ["Z"], "http://central:9000"));
|
||||
|
||||
// A warning mentioning the orphan name must be logged.
|
||||
await AwaitAssertAsync(() =>
|
||||
{
|
||||
Assert.Contains(logger.Entries, e =>
|
||||
e.Level == LogLevel.Warning && e.Message.Contains("Z"));
|
||||
return Task.CompletedTask;
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
// ...and Z is still present (never deleted) and no fetch happened.
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.Contains(configs, c => c.InstanceUniqueName == "Z");
|
||||
Assert.Empty(fetcher.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NoGap_DoesNotFetch()
|
||||
{
|
||||
// Empty gap and no orphans → the fetcher is never called.
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("never"));
|
||||
var commProbe = CreateTestProbe();
|
||||
|
||||
var actor = CreateReconciliationActor(commProbe, fetcher);
|
||||
|
||||
commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
||||
commProbe.Reply(new ReconcileSiteResponse([], [], "http://central:9000"));
|
||||
|
||||
// Give any (erroneous) continuation time to run, then prove no fetch happened.
|
||||
Watch(actor);
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
Assert.Empty(fetcher.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PerItemFetchFailure_DoesNotAbortTheRest()
|
||||
{
|
||||
// Gap = [Bad, Good]; the Bad fetch throws but Good must still be fetched + stored.
|
||||
const string configGood = "{\"instanceUniqueName\":\"Good\"}";
|
||||
var fetcher = new FakeConfigFetcher(depId => depId == "depBad"
|
||||
? Task.FromException<string>(new DeploymentConfigFetchException("boom", isSuperseded: false))
|
||||
: Task.FromResult(configGood));
|
||||
var commProbe = CreateTestProbe();
|
||||
|
||||
var actor = CreateReconciliationActor(commProbe, fetcher);
|
||||
Watch(actor);
|
||||
|
||||
commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
||||
commProbe.Reply(new ReconcileSiteResponse(
|
||||
[
|
||||
new ReconcileGapItem("Bad", "depBad", "revBad", true, "tok-bad"),
|
||||
new ReconcileGapItem("Good", "depGood", "revGood", true, "tok-good")
|
||||
],
|
||||
[],
|
||||
"http://central:9000"));
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
var row = Assert.Single(configs, c => c.InstanceUniqueName == "Good");
|
||||
Assert.Equal(configGood, row.ConfigJson);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
// Bad was attempted but never stored; the actor survived the per-item failure.
|
||||
var all = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.DoesNotContain(all, c => c.InstanceUniqueName == "Bad");
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SupersededItem_IsSkippedQuietly_OthersStillApply()
|
||||
{
|
||||
// Gap = [Gone, Good]; the Gone fetch throws a 404 (superseded/expired). That branch is
|
||||
// a quiet skip (logged Info, NOT counted as a failure) — the Good item must still apply
|
||||
// and the actor must not crash.
|
||||
const string configGood = "{\"instanceUniqueName\":\"Good\"}";
|
||||
var fetcher = new FakeConfigFetcher(depId => depId == "depGone"
|
||||
? Task.FromException<string>(new DeploymentConfigFetchException("expired", isSuperseded: true))
|
||||
: Task.FromResult(configGood));
|
||||
var logger = new CapturingLogger<SiteReconciliationActor>();
|
||||
var commProbe = CreateTestProbe();
|
||||
|
||||
var actor = CreateReconciliationActor(commProbe, fetcher, logger);
|
||||
Watch(actor);
|
||||
|
||||
commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
||||
commProbe.Reply(new ReconcileSiteResponse(
|
||||
[
|
||||
new ReconcileGapItem("Gone", "depGone", "revGone", true, "tok-gone"),
|
||||
new ReconcileGapItem("Good", "depGood", "revGood", true, "tok-good")
|
||||
],
|
||||
[],
|
||||
"http://central:9000"));
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
var row = Assert.Single(configs, c => c.InstanceUniqueName == "Good");
|
||||
Assert.Equal(configGood, row.ConfigJson);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
// The superseded item was attempted but never stored, and was logged at Info as a skip
|
||||
// (not Error) — and the actor survived.
|
||||
var all = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.DoesNotContain(all, c => c.InstanceUniqueName == "Gone");
|
||||
Assert.Contains(logger.Entries, e =>
|
||||
e.Level == LogLevel.Information && e.Message.Contains("Gone"));
|
||||
Assert.DoesNotContain(logger.Entries, e =>
|
||||
e.Level == LogLevel.Error && e.Message.Contains("Gone"));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CentralAskFails_ActorSurvives()
|
||||
{
|
||||
// The SiteCommunicationActor probe never replies → the Ask times out. The actor must
|
||||
// log + survive (reconcile re-runs on the next startup), not crash.
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("never"));
|
||||
var logger = new CapturingLogger<SiteReconciliationActor>();
|
||||
var commProbe = CreateTestProbe();
|
||||
|
||||
var actor = CreateReconciliationActor(
|
||||
commProbe, fetcher, logger, askTimeout: TimeSpan.FromMilliseconds(300));
|
||||
Watch(actor);
|
||||
|
||||
// The request is sent...
|
||||
commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
|
||||
// ...but the probe does NOT reply, forcing an Ask timeout.
|
||||
|
||||
// The actor does not die (no Terminated within the window) and never fetched.
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(1));
|
||||
Assert.Empty(fetcher.Calls);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-test fake <see cref="IDeploymentConfigFetcher"/>: runs a per-deploymentId behavior
|
||||
/// (return config JSON or throw, as a Task) and records every call's coords thread-safely
|
||||
/// (the fetch runs on a pool thread).
|
||||
/// </summary>
|
||||
private sealed class FakeConfigFetcher : IDeploymentConfigFetcher
|
||||
{
|
||||
private readonly Func<string, Task<string>> _behavior;
|
||||
public ConcurrentQueue<(string BaseUrl, string DeploymentId, string Token)> Calls { get; } = new();
|
||||
|
||||
public FakeConfigFetcher(Func<string, Task<string>> behavior) => _behavior = behavior;
|
||||
|
||||
public async Task<string> FetchAsync(
|
||||
string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct)
|
||||
{
|
||||
Calls.Enqueue((centralFetchBaseUrl, deploymentId, token));
|
||||
await Task.Yield();
|
||||
return await _behavior(deploymentId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Thread-safe capturing logger so tests can assert on emitted warnings.</summary>
|
||||
private sealed class CapturingLogger<T> : ILogger<T>
|
||||
{
|
||||
public ConcurrentQueue<(LogLevel Level, string Message)> Entries { get; } = new();
|
||||
|
||||
IDisposable? ILogger.BeginScope<TState>(TState state) => null;
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(
|
||||
LogLevel logLevel, EventId eventId, TState state, Exception? exception,
|
||||
Func<TState, Exception?, string> formatter)
|
||||
=> Entries.Enqueue((logLevel, formatter(state, exception)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="SiteReplicationActor"/>'s notify-and-fetch config replication:
|
||||
/// the active node now replicates an id-only <see cref="ReplicateConfigDeploy"/> (no inline
|
||||
/// config JSON — killing the intra-site 128 KB frame trap), and the standby fetches the
|
||||
/// config from central over HTTP and writes it with the older-write guard.
|
||||
/// </summary>
|
||||
public class SiteReplicationActorTests : TestKit, IDisposable
|
||||
{
|
||||
// Cluster provider is required because SiteReplicationActor calls Cluster.Get in its ctor
|
||||
// and subscribes to cluster events in PreStart. We use the in-memory TestTransport (not
|
||||
// dot-netty) so no real socket is bound and no DNS lookup happens — the actor only needs
|
||||
// the cluster extension to load; these tests never form a real two-node cluster.
|
||||
private const string ClusterConfig = @"
|
||||
akka {
|
||||
actor { provider = cluster }
|
||||
remote {
|
||||
enabled-transports = [""akka.remote.test""]
|
||||
test {
|
||||
transport-class = ""Akka.Remote.Transport.TestTransport, Akka.Remote""
|
||||
applied-adapters = []
|
||||
registry-key = site-repl-test
|
||||
local-address = ""test://site-repl@localhost:1""
|
||||
maximum-payload-bytes = 128000b
|
||||
scheme-identifier = test
|
||||
}
|
||||
}
|
||||
cluster { roles = [""site-test""] }
|
||||
loglevel = WARNING
|
||||
}";
|
||||
|
||||
private const string SiteRole = "site-test";
|
||||
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly StoreAndForwardStorage _sfStorage;
|
||||
private readonly ReplicationService _replicationService;
|
||||
private readonly string _dbFile;
|
||||
private readonly string _sfDbFile;
|
||||
|
||||
public SiteReplicationActorTests() : base(ClusterConfig, "site-repl")
|
||||
{
|
||||
_dbFile = Path.Combine(Path.GetTempPath(), $"site-repl-test-{Guid.NewGuid():N}.db");
|
||||
_sfDbFile = Path.Combine(Path.GetTempPath(), $"site-repl-sf-{Guid.NewGuid():N}.db");
|
||||
|
||||
_storage = new SiteStorageService(
|
||||
$"Data Source={_dbFile}", NullLogger<SiteStorageService>.Instance);
|
||||
_storage.InitializeAsync().GetAwaiter().GetResult();
|
||||
|
||||
_sfStorage = new StoreAndForwardStorage(
|
||||
$"Data Source={_sfDbFile}", NullLogger<StoreAndForwardStorage>.Instance);
|
||||
_sfStorage.InitializeAsync().GetAwaiter().GetResult();
|
||||
|
||||
_replicationService = new ReplicationService(
|
||||
new StoreAndForwardOptions(), NullLogger<ReplicationService>.Instance);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
Shutdown();
|
||||
try { File.Delete(_dbFile); } catch { /* cleanup */ }
|
||||
try { File.Delete(_sfDbFile); } catch { /* cleanup */ }
|
||||
}
|
||||
|
||||
private IActorRef CreateReplicationActor(IDeploymentConfigFetcher fetcher) =>
|
||||
ActorOf(Props.Create(() => new SiteReplicationActor(
|
||||
_storage, _sfStorage, _replicationService, SiteRole,
|
||||
NullLogger<SiteReplicationActor>.Instance, fetcher)));
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyConfigDeploy_StandbyFetchesConfigAndGuardedWrites()
|
||||
{
|
||||
// The standby receives an id-only ApplyConfigDeploy; it fetches the config from
|
||||
// central using the message's coords, then guarded-writes the fetched config.
|
||||
const string configJson = "{\"instanceUniqueName\":\"Pump1\"}";
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult(configJson));
|
||||
var actor = CreateReplicationActor(fetcher);
|
||||
|
||||
actor.Tell(new ApplyConfigDeploy(
|
||||
"Pump1", "dep-100", "sha256:abc", true,
|
||||
"http://central:9000", "tok-xyz"));
|
||||
|
||||
// The continuation runs off-thread; await the guarded write landing.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
var row = Assert.Single(configs, c => c.InstanceUniqueName == "Pump1");
|
||||
Assert.Equal(configJson, row.ConfigJson);
|
||||
Assert.Equal("dep-100", row.DeploymentId);
|
||||
Assert.Equal("sha256:abc", row.RevisionHash);
|
||||
Assert.True(row.IsEnabled);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
// The fetcher was called with the message's coords.
|
||||
var call = Assert.Single(fetcher.Calls);
|
||||
Assert.Equal("http://central:9000", call.BaseUrl);
|
||||
Assert.Equal("dep-100", call.DeploymentId);
|
||||
Assert.Equal("tok-xyz", call.Token);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyConfigDeploy_Superseded404_SkipsWriteAndActorSurvives()
|
||||
{
|
||||
// A 404 (superseded/expired) surfaces as DeploymentConfigFetchException{IsSuperseded}.
|
||||
// The standby must skip the write, observe the exception (no crash), and stay alive.
|
||||
var fetcher = new FakeConfigFetcher(_ =>
|
||||
Task.FromException<string>(
|
||||
new DeploymentConfigFetchException("expired", isSuperseded: true)));
|
||||
var actor = CreateReplicationActor(fetcher);
|
||||
|
||||
actor.Tell(new ApplyConfigDeploy(
|
||||
"GonePump", "dep-stale", "sha256:gone", true,
|
||||
"http://central:9000", "tok-stale"));
|
||||
|
||||
// The fetch was attempted...
|
||||
await AwaitAssertAsync(() =>
|
||||
{
|
||||
Assert.Single(fetcher.Calls);
|
||||
return Task.CompletedTask;
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
// ...the actor did not crash (no Terminated to its watcher within the window)...
|
||||
Watch(actor);
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
// ...and nothing was written for the superseded instance.
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "GonePump");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyConfigDeploy_EmptyFetchCoords_SkipsFetchAndWrite()
|
||||
{
|
||||
// The direct DeployInstanceCommand cross-cluster wire path was retired in Task 14.
|
||||
// This tests the defensive guard: if empty coords arrive, the actor must skip quietly
|
||||
// — no FetchAsync("") call, no write — rather than erroring.
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("never"));
|
||||
var actor = CreateReplicationActor(fetcher);
|
||||
|
||||
actor.Tell(new ApplyConfigDeploy(
|
||||
"NoCoordsPump", "dep-direct", "sha256:nc", true,
|
||||
CentralFetchBaseUrl: "", FetchToken: ""));
|
||||
|
||||
// Give any (erroneous) async continuation time to run, then prove neither happened.
|
||||
Watch(actor);
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
Assert.Empty(fetcher.Calls);
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "NoCoordsPump");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReplicateConfigDeploy_MapsToIdOnlyApplyConfigDeploy_ForPeer()
|
||||
{
|
||||
// The outbound mapping must forward an id-only ApplyConfigDeploy carrying the fetch
|
||||
// coords (and NO inline config) to the peer.
|
||||
var probe = CreateTestProbe();
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("unused"));
|
||||
var actor = ActorOf(Props.Create(() => new ProbeForwardingReplicationActor(
|
||||
_storage, _sfStorage, _replicationService, SiteRole,
|
||||
NullLogger<SiteReplicationActor>.Instance, fetcher, probe.Ref)));
|
||||
|
||||
actor.Tell(new ReplicateConfigDeploy(
|
||||
"Pump2", "dep-200", "sha256:def", false,
|
||||
"http://central:9000", "tok-abc"));
|
||||
|
||||
var applied = probe.ExpectMsg<ApplyConfigDeploy>(TimeSpan.FromSeconds(3));
|
||||
Assert.Equal("Pump2", applied.InstanceName);
|
||||
Assert.Equal("dep-200", applied.DeploymentId);
|
||||
Assert.Equal("sha256:def", applied.RevisionHash);
|
||||
Assert.False(applied.IsEnabled);
|
||||
Assert.Equal("http://central:9000", applied.CentralFetchBaseUrl);
|
||||
Assert.Equal("tok-abc", applied.FetchToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test subclass exposing the peer send: <see cref="SiteReplicationActor.SendToPeer"/> is
|
||||
/// overridden to forward to a probe so the outbound mapping can be asserted without a real
|
||||
/// two-node cluster (a single-node TestKit has no peer address, so the real send is dropped).
|
||||
/// </summary>
|
||||
private sealed class ProbeForwardingReplicationActor : SiteReplicationActor
|
||||
{
|
||||
private readonly IActorRef _peerProbe;
|
||||
|
||||
public ProbeForwardingReplicationActor(
|
||||
SiteStorageService storage, StoreAndForwardStorage sfStorage,
|
||||
ReplicationService replicationService, string siteRole,
|
||||
ILogger<SiteReplicationActor> logger, IDeploymentConfigFetcher configFetcher,
|
||||
IActorRef peerProbe)
|
||||
: base(storage, sfStorage, replicationService, siteRole, logger, configFetcher)
|
||||
=> _peerProbe = peerProbe;
|
||||
|
||||
protected override void SendToPeer(object message) => _peerProbe.Tell(message, Self);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-test fake <see cref="IDeploymentConfigFetcher"/>: runs a per-deploymentId behavior
|
||||
/// (return config JSON or throw, as a Task — mirroring the real async HTTP fetcher) and
|
||||
/// records every call's coords thread-safely (the continuation runs on a pool thread).
|
||||
/// </summary>
|
||||
private sealed class FakeConfigFetcher : IDeploymentConfigFetcher
|
||||
{
|
||||
private readonly Func<string, Task<string>> _behavior;
|
||||
public ConcurrentQueue<(string BaseUrl, string DeploymentId, string Token)> Calls { get; } = new();
|
||||
|
||||
public FakeConfigFetcher(Func<string, Task<string>> behavior) => _behavior = behavior;
|
||||
|
||||
public async Task<string> FetchAsync(
|
||||
string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct)
|
||||
{
|
||||
Calls.Enqueue((centralFetchBaseUrl, deploymentId, token));
|
||||
await Task.Yield();
|
||||
return await _behavior(deploymentId);
|
||||
}
|
||||
}
|
||||
}
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using System.Net;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Deployment;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for <see cref="HttpDeploymentConfigFetcher"/> using a stub
|
||||
/// <see cref="HttpMessageHandler"/>. No live HTTP connections are made.
|
||||
/// </summary>
|
||||
public class HttpDeploymentConfigFetcherTests
|
||||
{
|
||||
private static HttpDeploymentConfigFetcher CreateFetcher(HttpMessageHandler handler)
|
||||
{
|
||||
var client = new HttpClient(handler);
|
||||
return new HttpDeploymentConfigFetcher(client, NullLogger<HttpDeploymentConfigFetcher>.Instance);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Happy path
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_200_ReturnsBody()
|
||||
{
|
||||
const string expectedBody = """{"instanceUniqueName":"test"}""";
|
||||
var handler = new StubHandler(HttpStatusCode.OK, expectedBody);
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
var result = await fetcher.FetchAsync("http://central", "dep-1", "tok-abc", CancellationToken.None);
|
||||
|
||||
Assert.Equal(expectedBody, result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_200_RequestUrlIsCorrect()
|
||||
{
|
||||
var handler = new StubHandler(HttpStatusCode.OK, "{}");
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
await fetcher.FetchAsync("http://central", "dep-abc", "tok", CancellationToken.None);
|
||||
|
||||
Assert.NotNull(handler.LastRequest);
|
||||
Assert.Equal(
|
||||
"http://central/api/internal/deployments/dep-abc/config",
|
||||
handler.LastRequest!.RequestUri!.ToString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_200_SetsDeploymentTokenHeader()
|
||||
{
|
||||
var handler = new StubHandler(HttpStatusCode.OK, "{}");
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
await fetcher.FetchAsync("http://central", "dep-1", "my-secret-token", CancellationToken.None);
|
||||
|
||||
Assert.NotNull(handler.LastRequest);
|
||||
Assert.True(handler.LastRequest!.Headers.Contains("X-Deployment-Token"));
|
||||
Assert.Equal("my-secret-token", handler.LastRequest.Headers.GetValues("X-Deployment-Token").Single());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_BaseUrlWithTrailingSlash_NoDoubleSlashInUrl()
|
||||
{
|
||||
var handler = new StubHandler(HttpStatusCode.OK, "{}");
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
await fetcher.FetchAsync("http://central/", "dep-1", "tok", CancellationToken.None);
|
||||
|
||||
Assert.NotNull(handler.LastRequest);
|
||||
var url = handler.LastRequest!.RequestUri!.ToString();
|
||||
Assert.DoesNotContain("//api", url);
|
||||
Assert.Equal("http://central/api/internal/deployments/dep-1/config", url);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_DeploymentIdNeedingEscaping_EscapesOnceInUrl()
|
||||
{
|
||||
var handler = new StubHandler(HttpStatusCode.OK, "{}");
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
// A deploymentId containing a space and a slash — both require escaping.
|
||||
await fetcher.FetchAsync("http://central", "dep a/b", "tok", CancellationToken.None);
|
||||
|
||||
Assert.NotNull(handler.LastRequest);
|
||||
// Uri.OriginalString preserves the escaped form as built (no normalization/decoding).
|
||||
var url = handler.LastRequest!.RequestUri!.OriginalString;
|
||||
Assert.Equal("http://central/api/internal/deployments/dep%20a%2Fb/config", url);
|
||||
// Not double-encoded: the percent-sign itself must not be re-escaped to %25.
|
||||
Assert.DoesNotContain("%25", url);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// 404 → IsSuperseded = true
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_404_ThrowsWithIsSupersededTrue()
|
||||
{
|
||||
var handler = new StubHandler(HttpStatusCode.NotFound, "Not found");
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<DeploymentConfigFetchException>(
|
||||
() => fetcher.FetchAsync("http://central", "dep-gone", "tok", CancellationToken.None));
|
||||
|
||||
Assert.True(ex.IsSuperseded);
|
||||
Assert.Contains("dep-gone", ex.Message);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Other HTTP errors → IsSuperseded = false
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_401_ThrowsWithIsSupersededFalse()
|
||||
{
|
||||
var handler = new StubHandler(HttpStatusCode.Unauthorized, "Unauthorized");
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<DeploymentConfigFetchException>(
|
||||
() => fetcher.FetchAsync("http://central", "dep-1", "bad-tok", CancellationToken.None));
|
||||
|
||||
Assert.False(ex.IsSuperseded);
|
||||
Assert.Contains("401", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_500_ThrowsWithIsSupersededFalse()
|
||||
{
|
||||
var handler = new StubHandler(HttpStatusCode.InternalServerError, "Server error");
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<DeploymentConfigFetchException>(
|
||||
() => fetcher.FetchAsync("http://central", "dep-1", "tok", CancellationToken.None));
|
||||
|
||||
Assert.False(ex.IsSuperseded);
|
||||
Assert.Contains("500", ex.Message);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Transport failure → DeploymentConfigFetchException (not raw HttpRequestException)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_TransportException_WrapsAsDeploymentConfigFetchException()
|
||||
{
|
||||
var handler = new ThrowingHandler(new HttpRequestException("Connection refused"));
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<DeploymentConfigFetchException>(
|
||||
() => fetcher.FetchAsync("http://central", "dep-1", "tok", CancellationToken.None));
|
||||
|
||||
Assert.False(ex.IsSuperseded);
|
||||
Assert.IsType<HttpRequestException>(ex.InnerException);
|
||||
Assert.Contains("Connection refused", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_HttpClientTimeout_WrapsAsTransportError_NotCancellation()
|
||||
{
|
||||
// HttpClient.Timeout surfaces as TaskCanceledException while the caller's ct is NOT cancelled.
|
||||
var handler = new ThrowingHandler(new TaskCanceledException("simulated HttpClient timeout"));
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<DeploymentConfigFetchException>(
|
||||
() => fetcher.FetchAsync("http://central", "dep-1", "tok", CancellationToken.None));
|
||||
|
||||
Assert.False(ex.IsSuperseded);
|
||||
Assert.IsType<TaskCanceledException>(ex.InnerException);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Cancellation propagates as-is
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_CancelledToken_PropagatesOperationCanceledException()
|
||||
{
|
||||
using var cts = new CancellationTokenSource();
|
||||
cts.Cancel();
|
||||
|
||||
var handler = new DelayHandler(); // never returns
|
||||
var fetcher = CreateFetcher(handler);
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
() => fetcher.FetchAsync("http://central", "dep-1", "tok", cts.Token));
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Stub handlers
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
private sealed class StubHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly HttpStatusCode _status;
|
||||
private readonly string _body;
|
||||
public HttpRequestMessage? LastRequest { get; private set; }
|
||||
|
||||
public StubHandler(HttpStatusCode status, string body)
|
||||
{
|
||||
_status = status;
|
||||
_body = body;
|
||||
}
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
|
||||
{
|
||||
LastRequest = request;
|
||||
var resp = new HttpResponseMessage(_status)
|
||||
{
|
||||
Content = new StringContent(_body)
|
||||
};
|
||||
return Task.FromResult(resp);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ThrowingHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly Exception _ex;
|
||||
public ThrowingHandler(Exception ex) => _ex = ex;
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
|
||||
=> throw _ex;
|
||||
}
|
||||
|
||||
private sealed class DelayHandler : HttpMessageHandler
|
||||
{
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, ct);
|
||||
return new HttpResponseMessage(HttpStatusCode.OK);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
@@ -194,4 +195,112 @@ public class SiteStorageServiceTests : IAsyncLifetime, IDisposable
|
||||
Assert.Empty(configs);
|
||||
Assert.Empty(overrides);
|
||||
}
|
||||
|
||||
// ── Task 13: StoreDeployedConfigIfNewerAsync (guarded standby write) ──
|
||||
|
||||
/// <summary>
|
||||
/// Seeds a deployed_configurations row with an explicit deployed_at timestamp using the same
|
||||
/// "O" format the service uses, so tests can establish deterministic older/newer/equal rows.
|
||||
/// </summary>
|
||||
private async Task SeedDeployedConfigAsync(
|
||||
string instanceName, string configJson, string deploymentId,
|
||||
string revisionHash, bool isEnabled, DateTimeOffset deployedAt)
|
||||
{
|
||||
await using var conn = new SqliteConnection($"Data Source={_dbFile}");
|
||||
await conn.OpenAsync();
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
INSERT INTO deployed_configurations
|
||||
(instance_unique_name, config_json, deployment_id, revision_hash, is_enabled, deployed_at)
|
||||
VALUES (@name, @json, @depId, @hash, @enabled, @deployedAt)
|
||||
ON CONFLICT(instance_unique_name) DO UPDATE SET
|
||||
config_json = excluded.config_json,
|
||||
deployment_id = excluded.deployment_id,
|
||||
revision_hash = excluded.revision_hash,
|
||||
is_enabled = excluded.is_enabled,
|
||||
deployed_at = excluded.deployed_at";
|
||||
cmd.Parameters.AddWithValue("@name", instanceName);
|
||||
cmd.Parameters.AddWithValue("@json", configJson);
|
||||
cmd.Parameters.AddWithValue("@depId", deploymentId);
|
||||
cmd.Parameters.AddWithValue("@hash", revisionHash);
|
||||
cmd.Parameters.AddWithValue("@enabled", isEnabled ? 1 : 0);
|
||||
cmd.Parameters.AddWithValue("@deployedAt", deployedAt.ToString("O"));
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StoreDeployedConfigIfNewer_NoExistingRow_Inserts()
|
||||
{
|
||||
var at = DateTimeOffset.UtcNow;
|
||||
|
||||
await _storage.StoreDeployedConfigIfNewerAsync(
|
||||
"Pump1", "{\"v\":1}", "dep-001", "sha256:aaa", isEnabled: true, deployedAtOverride: at);
|
||||
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.Single(configs);
|
||||
Assert.Equal("Pump1", configs[0].InstanceUniqueName);
|
||||
Assert.Equal("{\"v\":1}", configs[0].ConfigJson);
|
||||
Assert.Equal("dep-001", configs[0].DeploymentId);
|
||||
Assert.Equal("sha256:aaa", configs[0].RevisionHash);
|
||||
Assert.True(configs[0].IsEnabled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StoreDeployedConfigIfNewer_ExistingOlderRow_Overwrites()
|
||||
{
|
||||
var olderAt = new DateTimeOffset(2026, 1, 1, 10, 0, 0, TimeSpan.Zero);
|
||||
var newerAt = new DateTimeOffset(2026, 1, 1, 11, 0, 0, TimeSpan.Zero);
|
||||
|
||||
await SeedDeployedConfigAsync("Pump1", "{\"v\":1}", "dep-001", "sha256:aaa", true, olderAt);
|
||||
|
||||
await _storage.StoreDeployedConfigIfNewerAsync(
|
||||
"Pump1", "{\"v\":2}", "dep-002", "sha256:bbb", isEnabled: false, deployedAtOverride: newerAt);
|
||||
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.Single(configs);
|
||||
Assert.Equal("{\"v\":2}", configs[0].ConfigJson);
|
||||
Assert.Equal("dep-002", configs[0].DeploymentId);
|
||||
Assert.Equal("sha256:bbb", configs[0].RevisionHash);
|
||||
Assert.False(configs[0].IsEnabled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StoreDeployedConfigIfNewer_ExistingNewerRow_IsNoop()
|
||||
{
|
||||
var newerAt = new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero);
|
||||
var olderAt = new DateTimeOffset(2026, 5, 1, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
// Seed the row that is already newer than what the standby would write
|
||||
await SeedDeployedConfigAsync("Pump1", "{\"v\":2}", "dep-002", "sha256:bbb", false, newerAt);
|
||||
|
||||
// Guarded write with an older timestamp — must be a NO-OP
|
||||
await _storage.StoreDeployedConfigIfNewerAsync(
|
||||
"Pump1", "{\"v\":1}", "dep-001", "sha256:aaa", isEnabled: true, deployedAtOverride: olderAt);
|
||||
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.Single(configs);
|
||||
// The newer seeded row must survive unchanged
|
||||
Assert.Equal("{\"v\":2}", configs[0].ConfigJson);
|
||||
Assert.Equal("dep-002", configs[0].DeploymentId);
|
||||
Assert.Equal("sha256:bbb", configs[0].RevisionHash);
|
||||
Assert.False(configs[0].IsEnabled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StoreDeployedConfigIfNewer_EqualDeployedAt_IsNoop()
|
||||
{
|
||||
var at = new DateTimeOffset(2026, 3, 15, 9, 30, 0, TimeSpan.Zero);
|
||||
|
||||
await SeedDeployedConfigAsync("Pump1", "{\"v\":1}", "dep-001", "sha256:aaa", true, at);
|
||||
|
||||
// Guarded write with the IDENTICAL timestamp — must be a NO-OP (> not >=)
|
||||
await _storage.StoreDeployedConfigIfNewerAsync(
|
||||
"Pump1", "{\"v\":2}", "dep-002", "sha256:bbb", isEnabled: false, deployedAtOverride: at);
|
||||
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.Single(configs);
|
||||
// Original row preserved
|
||||
Assert.Equal("{\"v\":1}", configs[0].ConfigJson);
|
||||
Assert.Equal("dep-001", configs[0].DeploymentId);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user