Compare commits

...

26 Commits

Author SHA1 Message Date
Joseph Doherty 4fb754f9f6 docs(archreview): sync Status Board — PLAN-06 Complete/merged (3e964ac2); initiative 152/192
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:52:08 -04:00
Joseph Doherty 3e964ac2cb docs(archreview): sync PLAN-06 tasks.json — PLAN COMPLETE (24/24), terminal verification green (8 Playwright E2E env-only)
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:44:15 -04:00
Joseph Doherty 15f5bc412c docs(inbound-api): dual-key API-key rotation procedure (create -> migrate -> disable -> delete)
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:36:47 -04:00
Joseph Doherty 40088a2123 fix(notifications): validate Twilio AccountSid format at save — closes the un-escaped URI interpolation door
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:36:47 -04:00
Joseph Doherty 37cf7c1a44 fix(esg): cancellation-aware ErrorClassifier.IsTransient overload — caller-cancelled work can never classify transient
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:36:47 -04:00
Joseph Doherty 0ee9975111 feat(notifications): OAuth2 authority/scope management + CLI + UI surfaces
Additive UpdateSmtpConfigCommand params (OAuth2Authority/OAuth2Scope),
preserve-on-null handler application, CLI --oauth2-authority/--oauth2-scope
on smtp update, Central UI text inputs shown only for the OAuth2 auth type,
and the Component-NotificationService doc paragraph with M365 defaults.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:30:28 -04:00
Joseph Doherty 2e7be1e852 fix(esg): structured JSON AuthConfiguration (matches entity doc + UI placeholders) with legacy colon fallback — keys containing ':' no longer misparse
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:30:28 -04:00
Joseph Doherty b5e80d4c00 feat(inbound-api): machine-readable error codes per spec (incl. SITE_UNREACHABLE) + 415 for non-JSON bodies
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:30:28 -04:00
Joseph Doherty 761729b5d0 perf(esg): bound outbound response bodies (ResponseHeadersRead + MaxResponseBodyBytes cap; oversize = permanent failure)
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:20:28 -04:00
Joseph Doherty eabd2820eb feat(esg): --timeout-seconds on external-system create/update (management command + CLI, additive contract)
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:20:28 -04:00
Joseph Doherty 0b916ff0c5 fix(inbound-api): defer DI-scope disposal to handler completion and count abandoned executions — closes the timeout use-after-dispose
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:20:28 -04:00
Joseph Doherty 39e1ca3eae fix(notifications): deterministic lowest-Id SMTP config selection (+ multi-row warning); atomic OAuth2 token cache entry
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:14:36 -04:00
Joseph Doherty eb7a913d22 feat(esg): {param} path-template substitution with consumed-param removal — implements the spec'd /recipes/{id} form
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:14:36 -04:00
Joseph Doherty d0b27b8957 feat(inbound-api): InvalidateMethod seam purging handler + known-bad — consumer entry point for the script-artifact invalidation contract (plan 05)
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:14:36 -04:00
Joseph Doherty 9e60347bde feat(notifications): configurable OAuth2 authority + scope per SMTP configuration (defaults preserve M365) — entity + EF + migration + token service
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:07:44 -04:00
Joseph Doherty 178ae35308 feat(notifications): SMS notifications retry under SmsConfiguration.MaxRetries/RetryDelay (SMTP policy remains the Email + fallback policy) — wires the dead fields
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:07:44 -04:00
Joseph Doherty f0dd016207 feat(esg): TimeoutSeconds in ExternalSystem UI form + Component-ExternalSystemGateway doc sync — closes the spec-drift gap
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:07:44 -04:00
Joseph Doherty e3ef320359 feat(esg): TimeoutSeconds travels the artifact pipeline (additive contract) into site SQLite so site-side calls honor it
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:07:44 -04:00
Joseph Doherty 86d1a5cbf2 feat(esg): honor per-system TimeoutSeconds in InvokeHttpAsync (falls back to DefaultHttpTimeout)
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:07:44 -04:00
Joseph Doherty 899af63b2a docs(inbound-api): DB row is authoritative for method scripts — update broken-save warning + Direct SQL Warning for the revision-checked cache
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 04:07:44 -04:00
Joseph Doherty 16bab58d85 fix(inbound-api): revision-check compiled handler cache against the fresh ApiMethod row — heals failover/direct-SQL/known-bad staleness
Also updates the now-obsolete CompileAndRegister_NonCompilingUpdate test
(InboundScriptExecutorTests.cs, not in the plan's Files list) to assert the
deliberate DB-authoritative behavior: a broken save now 500s consistently on
every node instead of the stale delegate silently serving.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 03:57:49 -04:00
Joseph Doherty 2767e4bca1 feat(esg): per-system TimeoutSeconds column on ExternalSystemDefinition (spec Component-ExternalSystemGateway Timeout) — entity + EF + migration
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 03:57:49 -04:00
Joseph Doherty 153b82c880 docs(delmia): document at-least-once failover semantics + DelmiaRecipeDownload idempotency requirement
Manual verification (task step 3) skipped: this worktree cannot reach a
live environment to fetch the deployed DelmiaRecipeDownload script via
`scadabridge api-method get`. Documents only the mandated contract per
PLAN-06 Task 23; verifying the deployed script against it is deferred to
whoever has live environment access.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 03:52:04 -04:00
Joseph Doherty 102f5b60ee perf(host): compile inbound API methods in parallel at startup
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 03:49:53 -04:00
Joseph Doherty 839baa7880 fix(notifications): SMS adapter short-circuits on first transient — kills duplicate amplification and bounds the dispatch sweep
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 03:49:18 -04:00
Joseph Doherty aa1c1093e4 fix(inbound-api): endpoint filter hot-reads MaxRequestBodyBytes via IOptionsMonitor
Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 03:48:49 -04:00
63 changed files with 6236 additions and 253 deletions
+1 -1
View File
@@ -162,7 +162,7 @@ Other peers in the `scadaproj` family (see `scadaproj/CLAUDE.md` for details): `
- Dispatcher loop polls due rows, resolves the list, delivers via the typed adapter; transient failures retry to `Parked`, permanent failures park immediately.
- Site→central handoff is at-least-once: ack-after-persist plus insert-if-not-exists on `NotificationId`.
- No Akka replication — MS SQL is the HA store; daily purge of terminal rows after a configurable window (default 365 days).
- Notification Outbox retry reuses central SMTP max-retry-count and fixed interval.
- Notification Outbox retry reuses central SMTP max-retry-count and fixed interval for Email; Sms uses SmsConfiguration.MaxRetries/RetryDelay, falling back to the SMTP policy when unset.
- Cached calls (`ExternalSystem.CachedCall`, `Database.CachedWrite`) return a `TrackedOperationId` tracking handle, unified with `Notify.Send`'s existing tracking model (`Notify.Status` retained as a thin alias).
- A site-local operation tracking table (SQLite, alongside the S&F buffer) is the source of truth for cached-call status; `Tracking.Status(id)` reads it site-locally and authoritatively; terminal rows purged after a configurable window (default 7 days).
- Unified tracking status lifecycle `Pending → Retrying → Delivered / Parked / Failed / Discarded`; `Failed` = permanent failure (also returned synchronously to the calling script). No `Forwarding` state for cached calls.
+5 -3
View File
@@ -15,7 +15,7 @@ Every plan follows the TDD bite-sized-task format and ships a co-located `.tasks
| [PLAN-03](PLAN-03-site-runtime-dcl.md) | Site Runtime & DCL | 26 | 26 | ✅ Complete (all 26 tasks; merged to `main` `c7a0c922`) | 28 actionable: 25 fixed, 3 deferred with rationale |
| [PLAN-04](PLAN-04-data-audit-backbone.md) | Data & Audit Backbone | 26 | 26 | ✅ Complete (all 26 tasks; merged to `main` `dbd3ed2f` via fast-forward; branch removed) | 27 items: 22 fixed, 4 deferred, 1 won't-fix (incl. NodeB reconcile from report 08) |
| [PLAN-05](PLAN-05-templates-deployment-transport.md) | Templates, Deployment & Transport | 25 | 25 | ✅ Complete (all 25 tasks; merged to `main` `ec23db35` via fast-forward; worktree removed, branch deleted) | 33 line-items: 27 fixed, 6 deferred with rationale |
| [PLAN-06](PLAN-06-edge-integrations.md) | Edge Integrations | 24 | 0 | ⬜ Not started | 24 findings: 21 fixed, 3 deferred/won't-fix |
| [PLAN-06](PLAN-06-edge-integrations.md) | Edge Integrations | 24 | 24 | ✅ Complete (all 24 tasks; merged to `main` `3e964ac2` via fast-forward; worktree removed, branch deleted) | 24 findings: 21 fixed, 3 deferred/won't-fix |
| [PLAN-07](PLAN-07-ui-management-security.md) | UI, Management & Security | 33 | 3 | 🟨 In progress (P0: T1, T2, T3) | 24 items: 20 fixed, 3 partial (deferrals documented), 1 won't-fix |
| [PLAN-08](PLAN-08-conventions-tests.md) | Conventions & Tests | 11 | 1 | 🟨 In progress (P0: T1) | 13 covered, 4 owned by other plans, ~20 triaged into deferred-work register |
@@ -33,9 +33,11 @@ Status values: ⬜ Not started · 🟨 In progress · ✅ Complete · ⏸ Blocke
**Wave 3 — PLAN-05 COMPLETE (2026-07-10):** all 25 tasks delivered on worktree branch `worktree-archreview-wave3-plan05` and **merged to `main` `ec23db35` via fast-forward** (main was still at PLAN-04's `dbd3ed2f`, zero divergence; worktree removed, branch deleted). Templates/Deployment/Transport hardening across seven report-and-wait waves: import correctness spine (BundleImporter consumes `BaseTemplateName`/composition/inheritance edges + import-time acyclicity, `MinTimeBetweenRuns`/`ExecutionTimeoutSeconds`/`LockedInDerived`/native-alarm-sources/real `AreaName` carried through — T1–T7), a reflection round-trip equivalence guard that surfaced and fixed 5 silent-data-loss holes (ES retry+methods-on-Add, notification recipients, folder parent edges — T8), flattener/collision-detector repeated-composition fixes (T9/T10), revision-hash + DiffService native-source coverage (T11/T12), native-source lock enforcement (T13), the `IScriptArtifactChangeBus` invalidation contract + Host in-process bus + Transport post-commit publisher (T14, plan-06 handoff), compile-verdict cache + read-only-path compile skip + post-success audit isolation + central NotDeployed-delete (T15–T18), and the closing security/quality chain: **import blocker severity split** (template-script findings advisory, ApiMethod hard — T19), **the script trust gate on bundle import** (5th `ScriptTrustValidator` call site, over script bodies *and* Expression-trigger bodies — T20, given a code-reviewer security pass that caught + fixed 2 Important gaps: expression-trigger coverage and per-origin local-declaration scoping), ScriptTrustPolicy deny-list hardening (T21), LineDiffer input cap (T22), ArtifactDiff placeholder-identity resolution (T23), the low-severity batch (session cap / full error lists / leaf-fallback warning — T24), and the design-doc + CLAUDE.md sweep (T25). **Terminal verification:** full slnx build 0 warn/0 err; the entire `dotnet test` sweep green (~5,900 unit + integration tests across every suite) plus a direct `CLI.Tests` run (341). Only residue: the same environmental 8 `CentralUI.PlaywrightTests` Deployment E2E failures (`SQLite Error 14` on `instance deploy` — needs a live docker cluster; PLAN-05 touched zero files in that project's deployment-DB path). Initiative total: **128 of 192** (sum of the Done column).
**Wave 4 — PLAN-06 COMPLETE (2026-07-10):** all 24 tasks delivered and **merged to `main` `3e964ac2` via fast-forward** (main was still at PLAN-05's `9b0cf44c`, zero divergence; worktree removed, branch deleted). Edge Integrations hardening across six report-and-wait waves of file-disjoint parallel subagents: the **InboundAPI** spine — revision-checked compiled-handler cache (heals failover / direct-SQL / known-bad staleness, DB row now authoritative — T1/T2), `InvalidateMethod` seam consuming PLAN-05's `IScriptArtifactChangeBus` contract (T3), deferred DI-scope disposal + orphan-execution telemetry (T4), machine-readable error `code`s incl. `SITE_UNREACHABLE` + 415-on-non-JSON (T5), `IOptionsMonitor` hot-reload (T6), parallel startup compile (T7); the **ESG** per-system `TimeoutSeconds` full vertical slice (entity→EF migration→artifact→site SQLite→client honor→mgmt→CLI→UI→doc — T8–T12) plus `{param}` path-template substitution (T13), bounded outbound response buffering (T14), structured JSON `AuthConfiguration` with legacy-colon fallback (T15), cancellation-aware `IsTransient` (T16); the **Notification** batch — SMS first-transient short-circuit (T17), wired the dead `SmsConfiguration.MaxRetries/RetryDelay` per-type retry policy (T18), configurable OAuth2 authority/scope (entity→EF migration→token service→mgmt/CLI/UI surfaces — T19/T20), deterministic lowest-Id SMTP selection + atomic OAuth2 token cache (T21), Twilio `AccountSid` format validation (T22); and the docs (DelmiaNotifier at-least-once semantics — T23; dual-key API-key rotation procedure — T24). Two EF migrations added (`AddExternalSystemTimeoutSeconds` T8, `AddSmtpOAuth2AuthorityScope` T19, serialized on the shared snapshot). **Terminal verification:** full slnx build 0 warn/0 err; every unit + integration suite green (InboundAPI 253, ESG 132, NotificationOutbox 132, NotificationService 57, ManagementService 245, CLI 346, CentralUI 893, SiteRuntime 494, Commons 597, ConfigDB 338, Host 260, + all others). Only residue: the same environmental 8 `CentralUI.PlaywrightTests` Deployment E2E failures (`SQLite Error 14` on `instance deploy` — needs a live docker cluster; PLAN-06 touched zero files in that project's deployment-DB path). Execution note: `isolation:"worktree"` subagents proved non-isolating here (shared index → commit races); the working model was pre-created dedicated `git worktree`s per task + cherry-pick onto an integration branch + per-wave combined build/impacted-suite barrier. Initiative total: **152 of 192** (sum of the Done column).
## Parallelization Map — Incomplete Tasks (2026-07-09)
Covers the plans with open tasks: PLAN-06 (24), PLAN-07 (30), PLAN-08 (10). PLAN-01/02/03/04/05 are complete — nothing to parallelize there (PLAN-05 finished 2026-07-10; the section below is retained for its cross-plan mutex analysis, which still governs PLAN-06–08). **"Ready"** = every `blockedBy` entry is already `completed` in the plan's `.tasks.json`, so the task can start now. Ready sets are recomputed below from the live manifests; re-derive after each merge.
Covers the plans with open tasks: PLAN-07 (30), PLAN-08 (10). PLAN-01/02/03/04/05/06 are complete — nothing to parallelize there (PLAN-05 finished 2026-07-10; PLAN-06 finished 2026-07-10; the section below is retained for its cross-plan mutex analysis, which still governs PLAN-07–08). **"Ready"** = every `blockedBy` entry is already `completed` in the plan's `.tasks.json`, so the task can start now. Ready sets are recomputed below from the live manifests; re-derive after each merge.
### Cross-plan serialization mutexes (a task holding one must be the ONLY task touching that surface, anywhere)
@@ -56,7 +58,7 @@ Covers the plans with open tasks: PLAN-06 (24), PLAN-07 (30), PLAN-08 (10). PLAN
- **PLAN-04** — ✅ COMPLETE (2026-07-09); all 26 tasks done, merged to `main` `dbd3ed2f`. No longer a parallelization lane.
- **PLAN-05** — ✅ COMPLETE (2026-07-10); all 25 tasks done, merged to `main` `ec23db35` (fast-forward). No longer a parallelization lane. (Historical: the BundleImporter serial chain T1→2→…→6→19→20→23 ran in-worktree; T14's `IScriptArtifactChangeBus` contract is shipped for PLAN-06 to consume.)
- **PLAN-06** — ready: **{1, 6, 7, 8, 17, 23}**. Lanes: **T8** is the sole migration in flight; **T7** holds the `Host/Program.cs` mutex (coordinate with P08 T4); **T1, T6, T17, T23** are free-concurrent (InboundScriptExecutor / InboundApiEndpointFilter / SmsAdapter / DelmiaNotifier docs). The ExternalSystem (T9→…), Notification (T18/T19→…) and inbound-contract (T2→T3→T4→T5) chains stay blocked.
- **PLAN-06** — ✅ COMPLETE (2026-07-10); all 24 tasks done, merged to `main` `3e964ac2` (fast-forward). No longer a parallelization lane. (Historical: ran as 6 waves — W1{1,6,7,8,17,23} W2{2,9,10,12,18,19} W3{3,13,21} W4{4,11,14} W5{5,15,20} W6{16,22,24}; the ExternalSystemClient chain T9→13→14→15→16 and inbound-executor chain T1→3→4→5 were the two serial spines.)
- **PLAN-07** — ready non-ManagementActor (mutually concurrent, file-disjoint): **{4, 6, 14, 19, 21, 26, 29 | 32}** (ArtifactDeploymentService / ConfigSecretScrubber / SecuredWriteRepository / ManagementEndpoints+options / Commons Browse-Cert message records / Security LoginThrottle / CentralUI PollGate — T29 and T32 share `AlarmSummary.razor`, so serialize those two). Ready **ManagementActor** tasks (→ the ONE global Lane M): **{8, 9, 10, 12, 28, 30, 31}** — **start with T10** (it unblocks T11, T13, T22, T23). **T33 = skip (dedupe).**
- **PLAN-08** — ready: **{2, 3, 4, 5, 7, 8, 9, 11}** — each is a new per-component validator file or an isolated test/doc, so **mutually parallel**, with one caveat: **T4 shares `Host/Program.cs` with P06 T7** (sequence them). This is the initiative's single highest-parallelism slice (independent validator files). T6, T10 stay blocked.
@@ -1,30 +1,30 @@
{
"planPath": "archreview/plans/PLAN-06-edge-integrations.md",
"tasks": [
{ "id": 1, "subject": "Task 1: Revision-checked inbound handler cache (self-healing across failover / direct SQL / known-bad)", "status": "pending", "blockedBy": [] },
{ "id": 2, "subject": "Task 2: DB-authoritative compile semantics — management warning text + design doc", "status": "pending", "blockedBy": [1] },
{ "id": 3, "subject": "Task 3: InvalidateMethod seam — the Inbound API consumer of plan 05's invalidation contract", "status": "pending", "blockedBy": [1, 2] },
{ "id": 4, "subject": "Task 4: Abandoned inbound executions — defer DI-scope disposal + orphan telemetry", "status": "pending", "blockedBy": [3] },
{ "id": 5, "subject": "Task 5: Inbound error contract — code field + SITE_UNREACHABLE + 415 for non-JSON bodies", "status": "pending", "blockedBy": [4, 6] },
{ "id": 6, "subject": "Task 6: InboundApiEndpointFilter hot-reads options via IOptionsMonitor", "status": "pending", "blockedBy": [] },
{ "id": 7, "subject": "Task 7: Parallelize startup compile of inbound methods", "status": "pending", "blockedBy": [] },
{ "id": 8, "subject": "Task 8: ExternalSystemDefinition.TimeoutSeconds — entity + EF mapping + migration", "status": "pending", "blockedBy": [] },
{ "id": 9, "subject": "Task 9: Honor the per-system timeout in ExternalSystemClient.InvokeHttpAsync", "status": "pending", "blockedBy": [8] },
{ "id": 10, "subject": "Task 10: Carry TimeoutSeconds through the artifact pipeline to sites", "status": "pending", "blockedBy": [8] },
{ "id": 11, "subject": "Task 11: TimeoutSeconds management commands + CLI", "status": "pending", "blockedBy": [8, 3] },
{ "id": 12, "subject": "Task 12: TimeoutSeconds in Central UI form + spec doc sync", "status": "pending", "blockedBy": [8] },
{ "id": 13, "subject": "Task 13: Path-template substitution (/recipes/{id}) in BuildUrl", "status": "pending", "blockedBy": [9] },
{ "id": 14, "subject": "Task 14: Bound outbound HTTP response buffering", "status": "pending", "blockedBy": [13] },
{ "id": 15, "subject": "Task 15: Structured JSON AuthConfiguration with legacy colon fallback", "status": "pending", "blockedBy": [14] },
{ "id": 16, "subject": "Task 16: Cancellation-aware ErrorClassifier.IsTransient overload", "status": "pending", "blockedBy": [15] },
{ "id": 17, "subject": "Task 17: SMS first-transient short-circuit (duplicate amplification + sweep bound)", "status": "pending", "blockedBy": [] },
{ "id": 18, "subject": "Task 18: Honor SmsConfiguration.MaxRetries/RetryDelay — per-type retry policy", "status": "pending", "blockedBy": [] },
{ "id": 19, "subject": "Task 19: Configurable OAuth2 authority/scope — entity + EF + migration + token service", "status": "pending", "blockedBy": [8] },
{ "id": 20, "subject": "Task 20: OAuth2 authority/scope surfaces — management command + CLI + UI + doc", "status": "pending", "blockedBy": [19] },
{ "id": 21, "subject": "Task 21: Notification low-severity cleanups — deterministic SMTP selection + OAuth2 cache visibility", "status": "pending", "blockedBy": [18, 19] },
{ "id": 22, "subject": "Task 22: Validate Twilio AccountSid format at config save", "status": "pending", "blockedBy": [11] },
{ "id": 23, "subject": "Task 23: Document DelmiaNotifier at-least-once semantics + idempotency requirement", "status": "pending", "blockedBy": [] },
{ "id": 24, "subject": "Task 24: Document the dual-key API-key rotation procedure", "status": "pending", "blockedBy": [2, 5] }
{ "id": 1, "subject": "Task 1: Revision-checked inbound handler cache (self-healing across failover / direct SQL / known-bad)", "status": "completed", "blockedBy": [] },
{ "id": 2, "subject": "Task 2: DB-authoritative compile semantics — management warning text + design doc", "status": "completed", "blockedBy": [1] },
{ "id": 3, "subject": "Task 3: InvalidateMethod seam — the Inbound API consumer of plan 05's invalidation contract", "status": "completed", "blockedBy": [1, 2] },
{ "id": 4, "subject": "Task 4: Abandoned inbound executions — defer DI-scope disposal + orphan telemetry", "status": "completed", "blockedBy": [3] },
{ "id": 5, "subject": "Task 5: Inbound error contract — code field + SITE_UNREACHABLE + 415 for non-JSON bodies", "status": "completed", "blockedBy": [4, 6] },
{ "id": 6, "subject": "Task 6: InboundApiEndpointFilter hot-reads options via IOptionsMonitor", "status": "completed", "blockedBy": [] },
{ "id": 7, "subject": "Task 7: Parallelize startup compile of inbound methods", "status": "completed", "blockedBy": [] },
{ "id": 8, "subject": "Task 8: ExternalSystemDefinition.TimeoutSeconds — entity + EF mapping + migration", "status": "completed", "blockedBy": [] },
{ "id": 9, "subject": "Task 9: Honor the per-system timeout in ExternalSystemClient.InvokeHttpAsync", "status": "completed", "blockedBy": [8] },
{ "id": 10, "subject": "Task 10: Carry TimeoutSeconds through the artifact pipeline to sites", "status": "completed", "blockedBy": [8] },
{ "id": 11, "subject": "Task 11: TimeoutSeconds management commands + CLI", "status": "completed", "blockedBy": [8, 3] },
{ "id": 12, "subject": "Task 12: TimeoutSeconds in Central UI form + spec doc sync", "status": "completed", "blockedBy": [8] },
{ "id": 13, "subject": "Task 13: Path-template substitution (/recipes/{id}) in BuildUrl", "status": "completed", "blockedBy": [9] },
{ "id": 14, "subject": "Task 14: Bound outbound HTTP response buffering", "status": "completed", "blockedBy": [13] },
{ "id": 15, "subject": "Task 15: Structured JSON AuthConfiguration with legacy colon fallback", "status": "completed", "blockedBy": [14] },
{ "id": 16, "subject": "Task 16: Cancellation-aware ErrorClassifier.IsTransient overload", "status": "completed", "blockedBy": [15] },
{ "id": 17, "subject": "Task 17: SMS first-transient short-circuit (duplicate amplification + sweep bound)", "status": "completed", "blockedBy": [] },
{ "id": 18, "subject": "Task 18: Honor SmsConfiguration.MaxRetries/RetryDelay — per-type retry policy", "status": "completed", "blockedBy": [] },
{ "id": 19, "subject": "Task 19: Configurable OAuth2 authority/scope — entity + EF + migration + token service", "status": "completed", "blockedBy": [8] },
{ "id": 20, "subject": "Task 20: OAuth2 authority/scope surfaces — management command + CLI + UI + doc", "status": "completed", "blockedBy": [19] },
{ "id": 21, "subject": "Task 21: Notification low-severity cleanups — deterministic SMTP selection + OAuth2 cache visibility", "status": "completed", "blockedBy": [18, 19] },
{ "id": 22, "subject": "Task 22: Validate Twilio AccountSid format at config save", "status": "completed", "blockedBy": [11] },
{ "id": 23, "subject": "Task 23: Document DelmiaNotifier at-least-once semantics + idempotency requirement", "status": "completed", "blockedBy": [] },
{ "id": 24, "subject": "Task 24: Document the dual-key API-key rotation procedure", "status": "completed", "blockedBy": [2, 5] }
],
"lastUpdated": "2026-07-08"
"lastUpdated": "2026-07-10"
}
@@ -105,6 +105,30 @@ response came back). A node that responds at all is authoritative — its answer
> to the next node — failover is strictly for unreachable nodes. (Revisit only if operations want
> `5xx`/`503` to also fail over.)
## Delivery semantics: at-least-once
The connect-failure-only failover loop above is by design (no code change here) — but its
consequence is at-least-once delivery of the `DelmiaRecipeDownload` POST, not exactly-once. Two
cases both count as "no response" from the client's point of view, trigger failover to the next
URL, and produce a duplicate POST even though the first node may have already processed the
request:
1. **Connection reset after the server processed the POST** — the request reaches the site and is
applied, but the response never makes it back to the client (dropped connection, reset). The
client sees a connect failure (row 1 of the table above), rolls over, and re-POSTs to the next
`BaseUrls` entry.
2. **Slow response exceeding `TimeoutSeconds`** — the server is still processing (or has already
finished) when the client's `HttpClient.Timeout` fires. The client treats the timeout the same
as an unreachable node, rolls over, and re-POSTs.
**Requirement:** the deployed `DelmiaRecipeDownload` inbound method **MUST be idempotent** on the
natural key `(MachineCode, DownloadPath, WorkOrderNumber)`. A duplicate POST for the same key must
return the same `Result`/`ResultText` as the original call, not double-apply side effects (e.g. not
queue the recipe download twice). This is a contract on the inbound method's script, not on the
DelmiaNotifier client — the client cannot distinguish a "duplicate delivered" retry from a
"first delivery" attempt, so idempotency must live on the receiving side. See the timeout row in
the failover table above for the triggering condition.
## Error handling & logging
- **stdout is reserved** for the `YES`/`NO` contract. All diagnostics — per-URL attempt, status code,
@@ -35,12 +35,18 @@ Each external system definition includes:
- **Authentication**: One of:
- **API Key**: Header name (e.g., `X-API-Key`) and key value.
- **Basic Auth**: Username and password.
- **Timeout**: Per-system timeout for all method calls (e.g., 30 seconds). Applies to the HTTP request round-trip.
The `AuthConfiguration` value is stored as **structured JSON** (the form the entity doc-comment and the Central UI placeholders promise):
- **API Key**: `{"header": "X-API-Key", "key": "<key>"}`. `header` is optional and defaults to `X-API-Key`; `key` is required. A key value containing `:` is preserved verbatim (JSON parsing does not split on `:`). Property names are matched case-insensitively.
- **Basic Auth**: `{"username": "<user>", "password": "<pass>"}`. Both fields are Base64-encoded as `username:password` for the `Authorization: Basic` header; a password containing `:` is preserved.
**Legacy colon form (deprecated):** a value that does not start with `{` is parsed with the historical colon-split rules for backward compatibility — API Key as `HeaderName:KeyValue` (or a bare `KeyValue` using the default `X-API-Key` header), Basic Auth as `username:password`. This form cannot represent keys or passwords that contain `:` and is retained only so existing rows keep working; new definitions should use the JSON form. Malformed JSON (a value that starts with `{` but does not parse, or yields no usable `key`/`username`) is **not** an exception — the request is sent without the auth header and a warning is logged (the configuration value itself is never logged).
- **Timeout**: Per-system timeout for all method calls (e.g., 30 seconds). Applies to the HTTP request round-trip. `0` (the default) means "unset" — the gateway's configured `DefaultHttpTimeout` applies instead. Set via the Central UI External System form's "Timeout (seconds)" field; ships to sites in the deployment artifact like the other definition fields.
- **Retry Settings**: Max retry count, fixed time between retries (used by Store-and-Forward Engine for transient failures only).
- **Method Definitions**: List of available API methods, each with:
- Method name.
- **HTTP method**: GET, POST, PUT, PATCH, or DELETE.
- **Path**: Relative path appended to the base URL (e.g., `/recipes/{id}`).
- **Path**: Relative path appended to the base URL (e.g., `/recipes/{id}`). A path may contain `{param}` placeholders (identifier-named — letter/underscore start, then letters/digits/underscores). At invocation each placeholder is replaced with the URL-escaped value of the like-named parameter, and that parameter is **consumed**: it is removed from both the GET/DELETE query string and the POST/PUT/PATCH JSON body so it is never duplicated. A placeholder with no matching parameter, or one whose value is null, is an authoring error and raises a clear `ArgumentException` naming the placeholder.
- Parameter definitions (name, type). Supports the extended type system (Boolean, Integer, Float, String, Object, List).
- Return type definition. Supports the extended type system for complex response structures.
@@ -72,7 +78,7 @@ Each database connection definition includes:
All external system calls are **HTTP/REST** with **JSON** serialization:
- The ESG acts as an HTTP client. The external system definition provides the base URL; each method definition specifies the HTTP method and relative path.
- Request parameters are serialized as JSON in the request body (POST/PUT/PATCH) or as query parameters (GET/DELETE).
- Request parameters are serialized as JSON in the request body (POST/PUT/PATCH) or as query parameters (GET/DELETE). Parameters consumed by `{param}` path-template substitution (see Method Definitions → Path) are carried in the URL path and excluded from both the body and the query string.
- Response bodies are deserialized from JSON into the method's defined return type.
- Credentials (API key header or Basic Auth header) are attached to every request per the system's authentication configuration.
@@ -98,7 +104,7 @@ Scripts choose between two call modes per invocation, mirroring the dual-mode da
## Call Timeout & Error Handling
- Each external system definition specifies a **timeout** that applies to all method calls on that system.
- Each external system definition specifies a **timeout** (`TimeoutSeconds`) that applies to all method calls on that system. `0` means unset — the gateway's configured `DefaultHttpTimeout` applies instead. Editable via the Central UI External System form ("Timeout (seconds)", helper text "0 = use gateway default") and deployed to sites in the artifact alongside the rest of the definition.
- Error classification by HTTP response:
- **Transient failures** (connection refused, timeout, HTTP 408, 429, 5xx): Behavior depends on call mode — `CachedCall` buffers for retry; `Call` returns error to script.
- **Permanent failures** (HTTP 4xx except 408/429): Always returned to the calling script regardless of call mode. Logged to Site Event Logging. For `CachedCall`, the failure is additionally recorded as a terminal `Failed` tracking record — so even a never-buffered cached call has an authoritative status record.
+34 -3
View File
@@ -47,6 +47,13 @@ Each API method definition includes:
- **Implementation Script**: C# script that executes when the method is called. Stored **inline** in the method definition. Follows standard C# authoring patterns but has no template inheritance — it is a standalone script tied to this method.
- **Timeout**: Configurable per method. Defines the maximum time the method is allowed to execute (including any routed calls to sites) before returning a timeout error to the caller.
> **Authoring note — idempotent methods for at-least-once callers**: some external callers retry
> on their own connect-failure/timeout failover loop and cannot tell "the first attempt actually
> succeeded" from "the first attempt was lost" — e.g. the `DelmiaRecipeDownload` method backing
> `DelmiaNotifier` (see `docs/plans/2026-06-26-delmia-recipe-notifier-design.md`, "Delivery
> semantics: at-least-once"). Implementation scripts for such methods must be idempotent on their
> natural key so a duplicate delivery returns the same result without double-applying side effects.
### Management
- Managed by users with the **Designer** role via the Central UI.
- All method definition changes are audit logged.
@@ -65,6 +72,15 @@ Each API method definition includes:
- When both headers are present, `Authorization` takes precedence.
- Both forms are processed by the same `IApiKeyVerifier` from the shared `ZB.MOM.WW.Auth.ApiKeys` library; the verifier strips the optional `Bearer ` prefix and performs the same peppered-HMAC constant-time verification regardless of which header was used.
### Key rotation
The `sbk_<keyId>_<secret>` design already supports zero-downtime rotation — a caller can hold two valid keys at once, so there is no moment where the old key must be cut off before the new one works. The procedure:
1. **Create a new key** with the same scope set as the key being rotated — `security api-key create --name <name> --methods <methods>` (CLI `security api-key` group). Scopes are **copied at creation only**: the operator must pass the same `--methods` list the old key was approved for; the CLI does not read the old key's scopes for you, so scope drift between the pair is an operator error, not something the system detects.
2. **Deliver the new token to the caller and confirm cutover.** The response from `create` surfaces the one-time bearer token (`PrintCreatedKey` — it is shown once and cannot be retrieved again, so capture it immediately). Both the old and new keys verify successfully during this window — that overlap is intended, not a bug, and gives the caller time to redeploy with the new token without a synchronized flag-day cutover. Confirm the caller has actually switched by checking the audit log's Actor field (recorded per call), which shows which key ID is serving live traffic.
3. **Disable — don't delete — the old key** once cutover is confirmed: `security api-key update --key-id <old-key-id> --enabled false`. Disabling is reversible (`--enabled true` re-enables it) if the cutover needs to be rolled back, whereas deletion is not.
4. **Delete the old key after a quarantine period** (an operator-judgment window to be sure nothing is still presenting the old token): `security api-key delete --key-id <old-key-id>`.
### Request Format
- Content-Type: `application/json`.
- Parameters are top-level JSON fields in the request body matching the method's parameter definitions:
@@ -88,7 +104,7 @@ Each API method definition includes:
]
}
```
- **Failure (4xx/5xx)**: The response body is an error object:
- **Failure (4xx/5xx)**: The response body is an error object carrying a human-readable `error` and a stable, machine-readable `code`:
```json
{
"error": "Site unreachable",
@@ -96,6 +112,21 @@ Each API method definition includes:
}
```
- HTTP status codes distinguish success from failure — no envelope wrapper.
- **Error codes.** Every failure body carries a `code` an external integrator can branch on (the `error` message is for humans and may change). The two authorization negatives — unknown method and key-not-in-scope — deliberately share one code (`NOT_APPROVED`) and one body so a caller cannot enumerate which method names exist:
| `code` | HTTP status | Meaning |
|--------|-------------|---------|
| `UNAUTHORIZED` | 401 | Missing/invalid API key (every auth-stage failure maps here — no stage leak). |
| `NOT_APPROVED` | 403 | Method not found **or** key not in scope — indistinguishable by design. |
| `UNSUPPORTED_MEDIA_TYPE` | 415 | Request carried a body with a non-JSON `Content-Type`. A body with **no** `Content-Type` is still parsed leniently as JSON. |
| `INVALID_JSON` | 400 | Request body was not valid JSON. |
| `VALIDATION_FAILED` | 400 | Parameter validation failed (missing required field, wrong type, undeclared field). |
| `BODY_TOO_LARGE` | 413 | Request body exceeded `MaxRequestBodyBytes`. |
| `STANDBY_NODE` | 503 | Request hit a standby central node; the inbound API serves only the active node. |
| `TIMEOUT` | 500 | The method script exceeded its execution timeout. |
| `SITE_UNREACHABLE` | 500 | A routed `Route.To(...).Call(...)` could not reach the target site. |
| `SCRIPT_COMPILE_FAILED` | 500 | The method's script failed to compile (returned consistently on every node until a compiling version is saved). |
| `SCRIPT_ERROR` | 500 | Catch-all for any other script-execution failure (details are logged centrally, never leaked to the caller). |
### Extended Type System
- API method parameter and return type definitions support an **extended type system** beyond the four template attribute types (Boolean, Integer, Float, String):
@@ -124,11 +155,11 @@ API method scripts are compiled at central startup — all method definitions ar
- Updating a method via the CLI (`api-method update --id <N> --code '...'`) or Management API triggers immediate recompilation (`CompileAndRegister`). The updated script takes effect on the next API call — no node restart is required.
- Creating a new method after startup: if the method is created but not yet compiled, the first invocation triggers lazy (on-demand) compilation.
> **Compile failures on save are non-fatal but surfaced.** `CompileAndRegister` only replaces the cached delegate when the new script compiles; on failure it leaves the **previously registered handler (if any) in place** — a broken save never takes a working method offline. The change still persists, so the Create/Update Management result carries a top-level **`compileWarning`** (null when the script compiled) listing the Roslyn/trust-model errors. On **update**, the previous version keeps serving while the broken script is *not* live; on **create**, the method returns `"Script compilation failed"` until a compiling version is saved. This means a saved script that does not compile is **not silently a no-op** — the caller (CLI / Central UI) sees the warning, and the real diagnostics are also logged centrally.
> **Compile failures on save are non-fatal but surfaced, and the DB row is authoritative.** The persisted `ApiMethod.Script` row is the single source of truth: on every request `ExecuteAsync` revision-checks the cached delegate against the freshly-fetched script and recompiles in place when it differs, so a saved-but-broken script does **not** leave a previously registered handler silently serving. The change still persists, so the Create/Update Management result carries a top-level **`compileWarning`** (null when the script compiled) listing the Roslyn/trust-model errors. On both **create** and **update**, once a non-compiling script is saved the method returns a compilation error (`HTTP 500`) **consistently on every central node** until a compiling version is saved — there is no node-divergent "old delegate still works here" state. This means a saved script that does not compile is **not silently a no-op** — the caller (CLI / Central UI) sees the warning, and the real diagnostics are also logged centrally.
### Direct SQL Warning
> **Do not edit API method scripts via direct SQL.** The in-memory compiled script will not be updated until the next node restart. Always use the CLI, Management API, or Central UI to modify API method scripts.
> **Direct SQL edits to API method scripts are now picked up automatically.** Because the DB row is authoritative and `ExecuteAsync` revision-checks the cached delegate against the freshly-fetched `ApiMethod.Script` on every request, a script changed directly in the database takes effect on the **next call** — no node restart and no Management round-trip is required. A broken direct edit surfaces the same way as a broken save: the method returns a `HTTP 500` compilation error consistently on all central nodes until a compiling version is stored. Editing via the CLI, Management API, or Central UI is still preferred because those paths validate the script up front and return the `compileWarning`, whereas a direct SQL edit is only diagnosed on the next request (and in the central log).
---
@@ -46,6 +46,8 @@ The SMTP configuration is defined centrally and used by the central Email delive
- **Authentication mode**: One of:
- **Basic Auth**: Username and password. For on-prem SMTP relays or servers that support basic authentication.
- **OAuth2 Client Credentials**: Tenant ID, Client ID, and Client Secret. For Microsoft 365 and other modern SMTP providers that require OAuth2. The Email adapter handles the token lifecycle internally (fetch, cache, refresh on expiry).
- **OAuth2 authority** (optional): The token-endpoint URL requested during the client-credentials grant. When omitted it defaults to the Microsoft 365 endpoint `https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token`, with the `{tenant}` placeholder substituted from the credential's tenant id; set it explicitly to target another identity provider (e.g. a non-M365 OAuth2 SMTP provider).
- **OAuth2 scope** (optional): The scope requested from the token endpoint. When omitted it defaults to the Microsoft 365 scope `https://outlook.office365.com/.default`; set it explicitly for another provider. Both fields are managed by Admin-role users via the CLI (`notification smtp update --oauth2-authority`/`--oauth2-scope`) and the Central UI `/notifications/smtp` page (the two inputs appear only for the OAuth2 auth type); a blank value is stored as null so the M365 default applies. Together they make "and other modern SMTP providers" true — a non-Microsoft OAuth2 relay is now configurable without a code change.
- **TLS mode**: None, StartTLS, or SSL.
- **From address**: The sender email address for all notifications (e.g., `scada-notifications@company.com`).
- **Connection timeout**: Maximum time to wait for SMTP connection (default: 30 seconds).
@@ -62,7 +64,7 @@ The `SmsConfiguration` entity is defined centrally and used by the central SMS d
- **Messaging Service SID** (optional): Twilio Messaging Service SID. When present, Twilio uses it for sender selection; used instead of the From number (so a Messaging-Service-only config needs no From number).
- **API base URL** (optional): Override for the Twilio REST API base URL (default: `https://api.twilio.com`). Allows pointing at a test/stub handler or a regional endpoint.
- **Connection timeout**: Maximum time to wait for a Twilio API response (honored per-send by the delivery adapter).
- **Max retries** / **Retry delay**: Reserved. The Notification Outbox dispatcher currently derives the retry policy (max-retries + interval) from the central SMTP configuration for all notification types; these per-SMS values are persisted and transported for forward-compatibility but are not yet read at dispatch time. Honoring them per-type is a deferred enhancement.
- **Max retries** / **Retry delay**: Live for SMS. The Notification Outbox dispatcher selects the retry policy by notification type — SMS notifications retry under these `SmsConfiguration` values, while Email (and every other type) reuses the central SMTP policy. When no SMS configuration row exists the dispatcher falls back to the SMTP policy, preserving the historical behavior until SMS is configured. A non-positive value is clamped to the outbox fallback (10 retries / 1-minute delay) with a Warning, matching the SMTP-policy clamp, so a misconfiguration cannot silently park SMS on the first transient failure.
The `SmsConfiguration` entity travels in Transport bundles — the Auth Token rides the encrypted `SecretsBlock` (keyed by Account SID), consistent with how SMTP credentials are bundled.
@@ -99,7 +101,7 @@ The `SmsNotificationDeliveryAdapter` delivers notifications to an SMS list via t
SMS has no BCC mechanism. The adapter sends one Twilio request per recipient and classifies each:
- **All accepted** → `Success`; `ResolvedTargets` is snapshotted with the accepted numbers.
- **Any transient failure** → `Transient` (the whole notification retries at the fixed interval, then Parks after max-retries). Numbers already accepted on a prior attempt are re-texted on retry — the same "re-send to all" characteristic the Email adapter already has with BCC. (v1 does not track per-recipient state; that is a documented future enhancement.)
- **Any transient failure** → `Transient` (the whole notification retries at the fixed interval, then Parks after max-retries). Numbers already accepted on a prior attempt are re-texted on retry — the same "re-send to all" characteristic the Email adapter already has with BCC. The recipient loop stops at the **first** transient rather than attempting the remaining recipients: every recipient accepted after the first transient would just be a guaranteed duplicate text on retry, and stopping early also bounds a black-holed endpoint to one request timeout per sweep instead of recipients × timeout. (v1 does not track per-recipient state; that is a documented future enhancement.)
- **No transient failures, mix of accepted + permanent-bad** → `Success` to the good numbers; permanently-bad numbers are recorded in `LastError`. The notification is not parked if anything got through.
- **All permanent / no recipients / no SMS config / list-not-found** → `Permanent` (Park).
@@ -49,6 +49,7 @@ public static class ExternalSystemCommands
var endpointUrlOption = new Option<string>("--endpoint-url") { Description = "Endpoint URL", Required = true };
var authTypeOption = new Option<string>("--auth-type") { Description = "Auth type", Required = true };
var authConfigOption = new Option<string?>("--auth-config") { Description = "Auth configuration JSON" };
var timeoutOption = new Option<int?>("--timeout-seconds") { Description = "Per-system HTTP timeout in seconds (0 = use gateway default); omit to preserve" };
var cmd = new Command("update") { Description = "Update an external system" };
cmd.Add(idOption);
@@ -56,6 +57,7 @@ public static class ExternalSystemCommands
cmd.Add(endpointUrlOption);
cmd.Add(authTypeOption);
cmd.Add(authConfigOption);
cmd.Add(timeoutOption);
cmd.SetAction(async (ParseResult result) =>
{
var id = result.GetValue(idOption);
@@ -63,9 +65,10 @@ public static class ExternalSystemCommands
var endpointUrl = result.GetValue(endpointUrlOption)!;
var authType = result.GetValue(authTypeOption)!;
var authConfig = result.GetValue(authConfigOption);
var timeoutSeconds = result.GetValue(timeoutOption);
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
new UpdateExternalSystemCommand(id, name, endpointUrl, authType, authConfig));
new UpdateExternalSystemCommand(id, name, endpointUrl, authType, authConfig, timeoutSeconds));
});
return cmd;
}
@@ -87,21 +90,24 @@ public static class ExternalSystemCommands
var endpointUrlOption = new Option<string>("--endpoint-url") { Description = "Endpoint URL", Required = true };
var authTypeOption = new Option<string>("--auth-type") { Description = "Auth type (ApiKey, BasicAuth)", Required = true };
var authConfigOption = new Option<string?>("--auth-config") { Description = "Auth configuration JSON" };
var timeoutOption = new Option<int?>("--timeout-seconds") { Description = "Per-system HTTP timeout in seconds (0 = use gateway default)" };
var cmd = new Command("create") { Description = "Create an external system" };
cmd.Add(nameOption);
cmd.Add(endpointUrlOption);
cmd.Add(authTypeOption);
cmd.Add(authConfigOption);
cmd.Add(timeoutOption);
cmd.SetAction(async (ParseResult result) =>
{
var name = result.GetValue(nameOption)!;
var endpointUrl = result.GetValue(endpointUrlOption)!;
var authType = result.GetValue(authTypeOption)!;
var authConfig = result.GetValue(authConfigOption);
var timeoutSeconds = result.GetValue(timeoutOption);
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
new CreateExternalSystemCommand(name, endpointUrl, authType, authConfig));
new CreateExternalSystemCommand(name, endpointUrl, authType, authConfig, timeoutSeconds));
});
return cmd;
}
@@ -142,6 +142,8 @@ public static class NotificationCommands
updateCmd.Add(SmtpFromOption);
updateCmd.Add(SmtpTlsModeOption);
updateCmd.Add(SmtpCredentialsOption);
updateCmd.Add(SmtpOAuth2AuthorityOption);
updateCmd.Add(SmtpOAuth2ScopeOption);
updateCmd.SetAction(async (ParseResult result) =>
{
return await CommandHelpers.ExecuteCommandAsync(
@@ -172,6 +174,18 @@ public static class NotificationCommands
Description = "SMTP credentials — 'username:password' for Basic, or client secret " +
"for OAuth2 (optional; preserves existing if omitted)",
};
private static readonly Option<string?> SmtpOAuth2AuthorityOption =
new("--oauth2-authority")
{
Description = "OAuth2 token-endpoint URL (optional; a '{tenant}' placeholder is " +
"substituted from the credential; preserves existing / M365 default if omitted)",
};
private static readonly Option<string?> SmtpOAuth2ScopeOption =
new("--oauth2-scope")
{
Description = "OAuth2 scope requested from the token endpoint " +
"(optional; preserves existing / M365 default if omitted)",
};
private static Option<string?> CreateTlsModeOption()
{
@@ -185,8 +199,9 @@ public static class NotificationCommands
/// <summary>
/// Builds the <see cref="UpdateSmtpConfigCommand"/> from a parsed <c>smtp update</c>
/// invocation. The optional <c>--tls-mode</c> / <c>--credentials</c> flags map to
/// null when omitted so the server-side handler preserves the existing values.
/// invocation. The optional <c>--tls-mode</c> / <c>--credentials</c> /
/// <c>--oauth2-authority</c> / <c>--oauth2-scope</c> flags map to null when omitted
/// so the server-side handler preserves the existing values.
/// </summary>
/// <param name="result">The parsed command-line result from the <c>smtp update</c> invocation.</param>
/// <returns>An <see cref="UpdateSmtpConfigCommand"/> populated from the parsed result.</returns>
@@ -199,7 +214,9 @@ public static class NotificationCommands
var from = result.GetValue(SmtpFromOption)!;
var tlsMode = result.GetValue(SmtpTlsModeOption);
var credentials = result.GetValue(SmtpCredentialsOption);
return new UpdateSmtpConfigCommand(id, server, port, authMode, from, tlsMode, credentials);
var oauth2Authority = result.GetValue(SmtpOAuth2AuthorityOption);
var oauth2Scope = result.GetValue(SmtpOAuth2ScopeOption);
return new UpdateSmtpConfigCommand(id, server, port, authMode, from, tlsMode, credentials, oauth2Authority, oauth2Scope);
}
private static Command BuildSms(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
@@ -62,6 +62,11 @@
<label class="form-label">Retry Delay (seconds)</label>
<input type="number" class="form-control" @bind="_retryDelaySeconds" min="0" />
</div>
<div class="mb-3">
<label class="form-label">Timeout (seconds)</label>
<input type="number" class="form-control" @bind="_timeoutSeconds" min="0" />
<div class="form-text">0 = use gateway default</div>
</div>
@if (_formError != null)
{
@@ -85,6 +90,7 @@
private string? _authConfig;
private int _maxRetries = 3;
private int _retryDelaySeconds = 5;
private int _timeoutSeconds;
private string? _formError;
private ExternalSystemDefinition? _existing;
@@ -112,6 +118,7 @@
_authConfig = _existing.AuthConfiguration;
_maxRetries = _existing.MaxRetries;
_retryDelaySeconds = (int)_existing.RetryDelay.TotalSeconds;
_timeoutSeconds = _existing.TimeoutSeconds;
}
}
catch (Exception ex) { _formError = ex.Message; }
@@ -138,6 +145,7 @@
_existing.AuthConfiguration = _authConfig?.Trim();
_existing.MaxRetries = _maxRetries;
_existing.RetryDelay = TimeSpan.FromSeconds(_retryDelaySeconds);
_existing.TimeoutSeconds = _timeoutSeconds;
await ExternalSystemRepository.UpdateExternalSystemAsync(_existing);
}
else
@@ -146,7 +154,8 @@
{
AuthConfiguration = _authConfig?.Trim(),
MaxRetries = _maxRetries,
RetryDelay = TimeSpan.FromSeconds(_retryDelaySeconds)
RetryDelay = TimeSpan.FromSeconds(_retryDelaySeconds),
TimeoutSeconds = _timeoutSeconds
};
await ExternalSystemRepository.AddExternalSystemAsync(es);
}
@@ -226,6 +226,15 @@
return;
}
// The AccountSid is interpolated un-escaped into the Twilio request URI, so reject
// any malformed SID client-side too (the ManagementActor enforces the same guard
// server-side; keep the message identical).
if (!System.Text.RegularExpressions.Regex.IsMatch(_accountSid.Trim(), "^AC[0-9a-fA-F]{32}$"))
{
_formError = "Account SID must match Twilio's format: 'AC' followed by 32 hex characters.";
return;
}
var typedAuthToken = string.IsNullOrWhiteSpace(_authToken) ? null : _authToken.Trim();
try
@@ -56,6 +56,13 @@
<div class="col-md-8">@smtp.FromAddress</div>
<div class="col-md-4 text-muted">Credentials</div>
<div class="col-md-8">@(string.IsNullOrWhiteSpace(smtp.Credentials) ? "(not set)" : "(stored)")</div>
@if (string.Equals(smtp.AuthType, "OAuth2", StringComparison.OrdinalIgnoreCase))
{
<div class="col-md-4 text-muted">OAuth2 Authority</div>
<div class="col-md-8">@(string.IsNullOrWhiteSpace(smtp.OAuth2Authority) ? "(M365 default)" : smtp.OAuth2Authority)</div>
<div class="col-md-4 text-muted">OAuth2 Scope</div>
<div class="col-md-8">@(string.IsNullOrWhiteSpace(smtp.OAuth2Scope) ? "(M365 default)" : smtp.OAuth2Scope)</div>
}
</div>
</div>
</div>
@@ -101,6 +108,21 @@
<input type="email" class="form-control" @bind="_fromAddress"
placeholder="noreply@example.com" />
</div>
@if (_authType == "OAuth2")
{
<div class="col-12">
<label class="form-label">OAuth2 Authority</label>
<input type="text" class="form-control" @bind="_oauth2Authority"
placeholder="https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token" />
<div class="form-text">Token-endpoint URL; leave blank for the Microsoft 365 default. A <code>{tenant}</code> placeholder is substituted from the credential.</div>
</div>
<div class="col-12">
<label class="form-label">OAuth2 Scope</label>
<input type="text" class="form-control" @bind="_oauth2Scope"
placeholder="https://outlook.office365.com/.default" />
<div class="form-text">Scope requested from the token endpoint; leave blank for the Microsoft 365 default.</div>
</div>
}
@if (_formError != null)
{
<div class="col-12"><div class="text-danger small">@_formError</div></div>
@@ -135,6 +157,8 @@
private string? _tlsMode;
private string? _credentials;
private string _fromAddress = string.Empty;
private string? _oauth2Authority;
private string? _oauth2Scope;
private string? _formError;
private ToastNotification _toast = default!;
@@ -168,6 +192,8 @@
_tlsMode = "None";
_credentials = null;
_fromAddress = string.Empty;
_oauth2Authority = null;
_oauth2Scope = null;
_formError = null;
_showForm = true;
}
@@ -181,6 +207,8 @@
_tlsMode = smtp.TlsMode;
_credentials = smtp.Credentials;
_fromAddress = smtp.FromAddress;
_oauth2Authority = smtp.OAuth2Authority;
_oauth2Scope = smtp.OAuth2Scope;
_formError = null;
_showForm = true;
}
@@ -210,6 +238,8 @@
_editingSmtp.TlsMode = _tlsMode;
_editingSmtp.Credentials = _credentials?.Trim();
_editingSmtp.FromAddress = _fromAddress.Trim();
_editingSmtp.OAuth2Authority = NormalizeOptional(_oauth2Authority);
_editingSmtp.OAuth2Scope = NormalizeOptional(_oauth2Scope);
await NotificationRepository.UpdateSmtpConfigurationAsync(_editingSmtp);
}
else
@@ -218,7 +248,9 @@
{
Port = _port,
TlsMode = _tlsMode,
Credentials = _credentials?.Trim()
Credentials = _credentials?.Trim(),
OAuth2Authority = NormalizeOptional(_oauth2Authority),
OAuth2Scope = NormalizeOptional(_oauth2Scope)
};
await NotificationRepository.AddSmtpConfigurationAsync(smtp);
}
@@ -232,4 +264,9 @@
_formError = ex.Message;
}
}
// Blank OAuth2 authority/scope means "use the Microsoft 365 default" — persist null,
// not an empty string, so the delivery adapter's default-substitution path fires.
private static string? NormalizeOptional(string? value)
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
}
@@ -17,6 +17,13 @@ public class ExternalSystemDefinition
/// <summary>Fixed delay between retry attempts.</summary>
public TimeSpan RetryDelay { get; set; }
/// <summary>
/// Per-system HTTP round-trip timeout in seconds for all method calls
/// (spec: Component-ExternalSystemGateway "Timeout"). 0 = unset — the gateway's
/// configured DefaultHttpTimeout applies.
/// </summary>
public int TimeoutSeconds { get; set; }
/// <summary>
/// Initializes a new <see cref="ExternalSystemDefinition"/>.
/// </summary>
@@ -28,19 +28,21 @@ public class SmsConfiguration
/// <summary>
/// Gets or sets the maximum number of delivery retries before parking.
/// <para>
/// RESERVED: the Notification Outbox dispatcher currently derives the retry policy
/// (max-retries + interval) from the central SMTP configuration for <em>all</em>
/// notification types — see the "retry reuses central SMTP max-retry-count and fixed
/// interval" design decision and <c>NotificationOutboxActor.ResolveRetryPolicyAsync</c>.
/// This per-SMS value is persisted/transported for forward-compatibility but is NOT
/// yet read at dispatch time. Honoring it per-type is a deferred enhancement (it would
/// supersede the shared-SMTP-policy decision and is not a silent behavioral change).
/// LIVE for SMS: the Notification Outbox dispatcher selects the retry policy by
/// notification <c>Type</c> — SMS notifications retry under this value (and
/// <see cref="RetryDelay"/>), while Email (and every other type) uses the central SMTP
/// policy. When no SMS configuration row exists the dispatcher falls back to the SMTP
/// policy. A non-positive value is clamped to the outbox fallback (10) with a Warning so
/// a misconfiguration cannot silently park SMS on the first transient failure. See
/// <c>NotificationOutboxActor.ResolveRetryPoliciesAsync</c>.
/// </para>
/// </summary>
public int MaxRetries { get; set; }
/// <summary>
/// Gets or sets the delay between retry attempts. RESERVED — see <see cref="MaxRetries"/>:
/// the dispatcher currently uses the shared SMTP-derived retry interval for all types.
/// Gets or sets the delay between retry attempts. LIVE for SMS — see
/// <see cref="MaxRetries"/>: the dispatcher schedules the next SMS attempt with this delay
/// (clamped to the 1-minute fallback if non-positive), falling back to the SMTP interval
/// when no SMS configuration exists.
/// </summary>
public TimeSpan RetryDelay { get; set; }
@@ -25,6 +25,20 @@ public class SmtpConfiguration
/// <summary>Gets or sets the delay between retry attempts.</summary>
public TimeSpan RetryDelay { get; set; }
/// <summary>
/// Gets or sets the OAuth2 token-endpoint authority template, or null to use the
/// Microsoft 365 default (<c>https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token</c>).
/// The stored value is the full token-endpoint URL with an optional <c>{tenant}</c>
/// placeholder substituted from the credential's tenant id.
/// </summary>
public string? OAuth2Authority { get; set; }
/// <summary>
/// Gets or sets the OAuth2 scope requested from the token endpoint, or null to use the
/// Microsoft 365 default (<c>https://outlook.office365.com/.default</c>).
/// </summary>
public string? OAuth2Scope { get; set; }
/// <summary>
/// Initializes a new <see cref="SmtpConfiguration"/> with required fields.
/// </summary>
@@ -5,4 +5,5 @@ public record ExternalSystemArtifact(
string EndpointUrl,
string AuthType,
string? AuthConfiguration,
string? MethodDefinitionsJson);
string? MethodDefinitionsJson,
int TimeoutSeconds = 0);
@@ -2,8 +2,8 @@ namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
public record ListExternalSystemsCommand;
public record GetExternalSystemCommand(int ExternalSystemId);
public record CreateExternalSystemCommand(string Name, string EndpointUrl, string AuthType, string? AuthConfiguration);
public record UpdateExternalSystemCommand(int ExternalSystemId, string Name, string EndpointUrl, string AuthType, string? AuthConfiguration);
public record CreateExternalSystemCommand(string Name, string EndpointUrl, string AuthType, string? AuthConfiguration, int? TimeoutSeconds = null);
public record UpdateExternalSystemCommand(int ExternalSystemId, string Name, string EndpointUrl, string AuthType, string? AuthConfiguration, int? TimeoutSeconds = null);
public record DeleteExternalSystemCommand(int ExternalSystemId);
// External System Methods
@@ -8,7 +8,7 @@ public record CreateNotificationListCommand(string Name, IReadOnlyList<string> R
public record UpdateNotificationListCommand(int NotificationListId, string Name, IReadOnlyList<string> RecipientEmails, NotificationType Type = NotificationType.Email, IReadOnlyList<string>? RecipientPhones = null);
public record DeleteNotificationListCommand(int NotificationListId);
public record ListSmtpConfigsCommand;
public record UpdateSmtpConfigCommand(int SmtpConfigId, string Server, int Port, string AuthMode, string FromAddress, string? TlsMode = null, string? Credentials = null);
public record UpdateSmtpConfigCommand(int SmtpConfigId, string Server, int Port, string AuthMode, string FromAddress, string? TlsMode = null, string? Credentials = null, string? OAuth2Authority = null, string? OAuth2Scope = null);
public record ListSmsConfigsCommand;
// FromNumber is optional: a Twilio Messaging-Service-only config supplies MessagingServiceSid
// instead. At-least-one-of (FromNumber, MessagingServiceSid) is validated at the CLI/UI boundary.
@@ -37,6 +37,11 @@ public static class ScadaBridgeTelemetry
Meter.CreateCounter<long>("scadabridge.inbound_api.requests", unit: "1",
description: "Inbound API requests, tagged by method.");
/// <summary>Incremented each time an inbound execution is abandoned (handler outlived its request), tagged with the API method.</summary>
private static readonly Counter<long> _inboundAbandonedExecutions =
Meter.CreateCounter<long>("scadabridge.inbound_api.abandoned_executions", unit: "1",
description: "Inbound API executions abandoned after timeout/abort (handler still running), tagged by method.");
/// <summary>Incremented each time an S&amp;F buffer replication op fails to dispatch/deliver to the peer.</summary>
private static readonly Counter<long> _replicationFailures =
Meter.CreateCounter<long>("scadabridge.store_and_forward.replication.failures", unit: "1",
@@ -75,6 +80,11 @@ public static class ScadaBridgeTelemetry
public static void RecordInboundApiRequest(string method) =>
_inboundApiRequests.Add(1, new KeyValuePair<string, object?>("method", method));
/// <summary>Records that an inbound execution was abandoned (handler outlived its request) for the given <paramref name="method"/>.</summary>
/// <param name="method">The API method whose execution was abandoned.</param>
public static void RecordInboundAbandonedExecution(string method) =>
_inboundAbandonedExecutions.Add(1, new KeyValuePair<string, object?>("method", method));
/// <summary>Records one failed S&amp;F replication dispatch.</summary>
public static void RecordReplicationFailure() => _replicationFailures.Add(1);
@@ -29,6 +29,8 @@ public class ExternalSystemDefinitionConfiguration : IEntityTypeConfiguration<Ex
builder.Property(e => e.AuthConfiguration)
.HasMaxLength(8000);
builder.Property(e => e.TimeoutSeconds).HasDefaultValue(0);
builder.HasMany<ExternalSystemMethod>()
.WithOne()
.HasForeignKey(m => m.ExternalSystemDefinitionId)
@@ -126,5 +126,14 @@ public class SmtpConfigurationConfiguration : IEntityTypeConfiguration<SmtpConfi
builder.Property(s => s.FromAddress)
.IsRequired()
.HasMaxLength(500);
// Optional OAuth2 token-endpoint overrides — null preserves the M365 defaults.
builder.Property(s => s.OAuth2Authority)
.IsRequired(false)
.HasMaxLength(500);
builder.Property(s => s.OAuth2Scope)
.IsRequired(false)
.HasMaxLength(500);
}
}
@@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
{
/// <inheritdoc />
public partial class AddExternalSystemTimeoutSeconds : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<int>(
name: "TimeoutSeconds",
table: "ExternalSystemDefinitions",
type: "int",
nullable: false,
defaultValue: 0);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "TimeoutSeconds",
table: "ExternalSystemDefinitions");
}
}
}
@@ -0,0 +1,40 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
{
/// <inheritdoc />
public partial class AddSmtpOAuth2AuthorityScope : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "OAuth2Authority",
table: "SmtpConfigurations",
type: "nvarchar(500)",
maxLength: 500,
nullable: true);
migrationBuilder.AddColumn<string>(
name: "OAuth2Scope",
table: "SmtpConfigurations",
type: "nvarchar(500)",
maxLength: 500,
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "OAuth2Authority",
table: "SmtpConfigurations");
migrationBuilder.DropColumn(
name: "OAuth2Scope",
table: "SmtpConfigurations");
}
}
}
@@ -418,6 +418,11 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
b.Property<TimeSpan>("RetryDelay")
.HasColumnType("time");
b.Property<int>("TimeoutSeconds")
.ValueGeneratedOnAdd()
.HasColumnType("int")
.HasDefaultValue(0);
b.HasKey("Id");
b.HasIndex("Name")
@@ -979,6 +984,14 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
b.Property<int>("MaxRetries")
.HasColumnType("int");
b.Property<string>("OAuth2Authority")
.HasMaxLength(500)
.HasColumnType("nvarchar(500)");
b.Property<string>("OAuth2Scope")
.HasMaxLength(500)
.HasColumnType("nvarchar(500)");
b.Property<int>("Port")
.HasColumnType("int");
@@ -52,6 +52,15 @@ WWNotifier.exe -m Z28061 -d "C:\recipes\job.nc" -w WO12345 -p PN-7788 -s 0100 -u
- All diagnostics (per-URL attempt, status code, which URL answered, exceptions) go to **stderr** and,
if `LogPath` is set, an appended log file — never to stdout.
## Delivery semantics
The `BaseUrls` failover loop is at-least-once, not exactly-once: a connection reset *after* the
target already processed the POST, or a slow response that exceeds `TimeoutSeconds`, both look like
an unreachable node to this client and cause a re-POST to the next URL — even though the first
attempt may already have succeeded. The deployed `DelmiaRecipeDownload` inbound method must be
idempotent on `(MachineCode, DownloadPath, WorkOrderNumber)` so a duplicate delivery is harmless;
see the design doc's "Delivery semantics: at-least-once" section for details.
## Configuration — `appsettings.json`
Placed next to the exe (copied to the output on build/publish):
@@ -192,7 +192,7 @@ public class ArtifactDeploymentService
}))
: null;
externalSystemArtifacts.Add(new ExternalSystemArtifact(
es.Name, es.EndpointUrl, es.AuthType, es.AuthConfiguration, methodsJson));
es.Name, es.EndpointUrl, es.AuthType, es.AuthConfiguration, methodsJson, es.TimeoutSeconds));
}
// Map database connections
@@ -26,6 +26,10 @@ public static class ErrorClassifier
/// <summary>
/// Determines whether an exception represents a transient failure.
/// Prefer the cancellation-aware <see cref="IsTransient(Exception, CancellationToken)"/>
/// overload at call sites that hold the caller's token — this token-less overload
/// classifies every <see cref="OperationCanceledException"/> as transient, including
/// caller-requested cancellations that must never be buffered for S&amp;F retry.
/// </summary>
/// <param name="exception">The exception to classify.</param>
/// <returns><see langword="true"/> for connection/timeout/cancellation exceptions; <see langword="false"/> otherwise.</returns>
@@ -37,6 +41,18 @@ public static class ErrorClassifier
or OperationCanceledException;
}
/// <summary>
/// Cancellation-aware overload: a caller-requested cancellation is NOT transient
/// (buffering caller-abandoned work into S&amp;F would be wrong). Prefer this overload
/// at call sites that hold the caller's token.
/// </summary>
public static bool IsTransient(Exception exception, CancellationToken cancellationToken)
{
if (exception is OperationCanceledException && cancellationToken.IsCancellationRequested)
return false;
return IsTransient(exception);
}
/// <summary>
/// Creates a TransientException for S&amp;F buffering.
/// </summary>
@@ -2,6 +2,7 @@ using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.ExternalSystems;
@@ -274,7 +275,7 @@ public class ExternalSystemClient : IExternalSystemClient
// Timeout property is per-instance.
client.Timeout = Timeout.InfiniteTimeSpan;
var url = BuildUrl(system.EndpointUrl, method.Path, parameters, method.HttpMethod);
var url = BuildUrl(system.EndpointUrl, method.Path, parameters, method.HttpMethod, out var consumedParams);
// The request and response own IDisposable resources (StringContent, the
// response content stream). Dispose both, including on the exception paths.
@@ -288,28 +289,45 @@ public class ExternalSystemClient : IExternalSystemClient
method.HttpMethod.Equals("PUT", StringComparison.OrdinalIgnoreCase) ||
method.HttpMethod.Equals("PATCH", StringComparison.OrdinalIgnoreCase))
{
if (parameters != null && parameters.Count > 0)
// Parameters consumed by `{param}` path-template substitution are
// already carried in the URL — exclude them from the JSON body so a
// path parameter is not duplicated as a body field.
var bodyParameters = consumedParams.Count == 0
? parameters
: parameters?.Where(p => !consumedParams.Contains(p.Key))
.ToDictionary(p => p.Key, p => p.Value);
if (bodyParameters != null && bodyParameters.Count > 0)
{
request.Content = new StringContent(
JsonSerializer.Serialize(parameters),
JsonSerializer.Serialize(bodyParameters),
Encoding.UTF8,
"application/json");
}
}
// Enforce the per-call timeout. ExternalSystemDefinition has no per-system
// Timeout field yet, so the configured DefaultHttpTimeout is the effective
// round-trip limit (the design's "timeout applies to the HTTP request
// round-trip" guarantee). A linked CTS lets us distinguish a timeout from a
// caller-initiated cancellation: only the timeout is reclassified as transient.
using var timeoutCts = new CancellationTokenSource(_options.DefaultHttpTimeout);
// Enforce the per-call timeout (the design's "timeout applies to the HTTP
// request round-trip" guarantee). Resolution order: a positive per-system
// ExternalSystemDefinition.TimeoutSeconds wins; otherwise the configured
// DefaultHttpTimeout is the effective round-trip limit. A linked CTS lets us
// distinguish a timeout from a caller-initiated cancellation: only the
// timeout is reclassified as transient.
var effectiveTimeout = system.TimeoutSeconds > 0
? TimeSpan.FromSeconds(system.TimeoutSeconds)
: _options.DefaultHttpTimeout;
using var timeoutCts = new CancellationTokenSource(effectiveTimeout);
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(
cancellationToken, timeoutCts.Token);
HttpResponseMessage response;
try
{
response = await client.SendAsync(request, linkedCts.Token);
// ResponseHeadersRead returns as soon as the headers are in, so the
// body is not eagerly buffered — the bounded read below streams it and
// aborts once MaxResponseBodyBytes is exceeded, keeping a hostile or
// misbehaving endpoint from inflating the active node's memory.
response = await client.SendAsync(
request, HttpCompletionOption.ResponseHeadersRead, linkedCts.Token);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
@@ -320,9 +338,9 @@ public class ExternalSystemClient : IExternalSystemClient
{
// Our own timeout elapsed — a transient failure per the design.
throw ErrorClassifier.AsTransient(
$"Timeout calling {system.Name} after {_options.DefaultHttpTimeout.TotalSeconds:0.##}s", ex);
$"Timeout calling {system.Name} after {effectiveTimeout.TotalSeconds:0.##}s", ex);
}
catch (Exception ex) when (ErrorClassifier.IsTransient(ex))
catch (Exception ex) when (ErrorClassifier.IsTransient(ex, cancellationToken))
{
throw ErrorClassifier.AsTransient($"Connection error to {system.Name}: {ex.Message}", ex);
}
@@ -334,7 +352,8 @@ public class ExternalSystemClient : IExternalSystemClient
string body;
try
{
body = await response.Content.ReadAsStringAsync(linkedCts.Token);
body = await ReadBodyBoundedAsync(
response.Content, system.Name, _options.MaxResponseBodyBytes, linkedCts.Token);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
@@ -343,7 +362,7 @@ public class ExternalSystemClient : IExternalSystemClient
catch (OperationCanceledException ex) when (timeoutCts.IsCancellationRequested)
{
throw ErrorClassifier.AsTransient(
$"Timeout reading response from {system.Name} after {_options.DefaultHttpTimeout.TotalSeconds:0.##}s", ex);
$"Timeout reading response from {system.Name} after {effectiveTimeout.TotalSeconds:0.##}s", ex);
}
if (response.IsSuccessStatusCode)
@@ -418,6 +437,51 @@ public class ExternalSystemClient : IExternalSystemClient
}
}
/// <summary>
/// Reads an HTTP response body into a string while enforcing an upper size
/// bound. When the <c>Content-Length</c> header is present it fails fast;
/// otherwise the body is streamed in 16 KiB chunks and the read is aborted the
/// moment the accumulated length exceeds <paramref name="maxBytes"/> (cap+1
/// pattern), so the whole oversized body is never buffered. Oversize is a
/// permanent failure — retrying will not shrink the response, and buffering it
/// into store-and-forward would defeat the cap. Cancellation
/// (<see cref="OperationCanceledException"/>) is allowed to propagate so the
/// caller's ordered timeout/cancellation catch filters classify it.
/// </summary>
private static async Task<string> ReadBodyBoundedAsync(
HttpContent content, string systemName, long maxBytes, CancellationToken token)
{
// Fast path: a declared Content-Length above the cap is rejected before a
// single body byte is read.
var declaredLength = content.Headers.ContentLength;
if (declaredLength.HasValue && declaredLength.Value > maxBytes)
{
throw new PermanentExternalSystemException(
$"Response from {systemName} exceeded the {maxBytes}-byte limit " +
$"(declared Content-Length {declaredLength.Value}).");
}
await using var stream = await content.ReadAsStreamAsync(token);
using var buffer = new MemoryStream();
var chunk = new byte[16 * 1024];
int read;
while ((read = await stream.ReadAsync(chunk.AsMemory(0, chunk.Length), token)) > 0)
{
buffer.Write(chunk, 0, read);
// cap+1: as soon as we hold more than maxBytes we know it is oversized,
// without reading the remainder of the stream.
if (buffer.Length > maxBytes)
{
throw new PermanentExternalSystemException(
$"Response from {systemName} exceeded the {maxBytes}-byte limit.");
}
}
return Encoding.UTF8.GetString(buffer.GetBuffer(), 0, (int)buffer.Length);
}
private static string Truncate(string value, int maxChars)
{
if (string.IsNullOrEmpty(value) || value.Length <= maxChars)
@@ -428,14 +492,56 @@ public class ExternalSystemClient : IExternalSystemClient
return value.Substring(0, maxChars) + $"… [truncated, {value.Length} chars total]";
}
private static string BuildUrl(string baseUrl, string path, IReadOnlyDictionary<string, object?>? parameters, string httpMethod)
/// <summary>
/// Matches a `{param}` path-template placeholder. The name must be a valid
/// identifier (letter/underscore start, then letters/digits/underscores) so the
/// pattern cannot accidentally swallow JSON-ish braces in a hand-authored path.
/// </summary>
private static readonly Regex PathParamRegex = new(
@"\{([A-Za-z_][A-Za-z0-9_]*)\}", RegexOptions.Compiled);
private static string BuildUrl(
string baseUrl,
string path,
IReadOnlyDictionary<string, object?>? parameters,
string httpMethod,
out ISet<string> consumedParams)
{
consumedParams = new HashSet<string>(StringComparer.Ordinal);
// A method that targets the base URL itself has an empty (or "/") path.
// Appending a trailing "/" in that case yields ".../api/" which some
// servers treat as a distinct resource — only append a segment when the
// method actually defines a non-empty relative path.
var trimmedBase = baseUrl.TrimEnd('/');
var trimmedPath = path.Trim().TrimStart('/');
// Substitute `{param}` placeholders in the path with escaped parameter
// values. Each substituted name is recorded so it is excluded from BOTH
// the query string (GET/DELETE) and the JSON body (POST/PUT/PATCH) — a
// path parameter must not be duplicated. A placeholder with no matching
// parameter (or a null value) is an authoring error: throw a clear
// ArgumentException naming it, mirroring ValidateHttpMethod.
if (trimmedPath.Length > 0 && trimmedPath.Contains('{'))
{
var consumed = consumedParams;
trimmedPath = PathParamRegex.Replace(trimmedPath, match =>
{
var name = match.Groups[1].Value;
if (parameters == null ||
!parameters.TryGetValue(name, out var value) ||
value == null)
{
throw new ArgumentException(
$"Path template parameter '{{{name}}}' has no value — supply a non-null '{name}' parameter for this method.",
nameof(path));
}
consumed.Add(name);
return Uri.EscapeDataString(value.ToString() ?? string.Empty);
});
}
var url = string.IsNullOrEmpty(trimmedPath)
? trimmedBase
: trimmedBase + "/" + trimmedPath;
@@ -445,8 +551,9 @@ public class ExternalSystemClient : IExternalSystemClient
httpMethod.Equals("DELETE", StringComparison.OrdinalIgnoreCase)) &&
parameters != null && parameters.Count > 0)
{
var consumed = consumedParams;
var queryString = string.Join("&",
parameters.Where(p => p.Value != null)
parameters.Where(p => p.Value != null && !consumed.Contains(p.Key))
.Select(p => $"{Uri.EscapeDataString(p.Key)}={Uri.EscapeDataString(p.Value?.ToString() ?? "")}"));
// Only append "?" when the effective query string is non-empty — a method
@@ -484,10 +591,40 @@ public class ExternalSystemClient : IExternalSystemClient
return;
}
// A config whose trimmed value starts with '{' is the structured JSON form
// promised by the entity doc-comment and the Central UI placeholders. Parse
// it first; only a non-JSON value falls through to the legacy colon-split.
var isJson = system.AuthConfiguration.TrimStart().StartsWith('{');
Dictionary<string, string?>? jsonFields = null;
var jsonParsed = isJson && TryParseJsonAuth(system.AuthConfiguration, out jsonFields);
switch (authType)
{
case "apikey":
// Auth config format: "HeaderName:KeyValue" or just "KeyValue" (default header: X-API-Key)
// JSON form: {"header"?: string, "key": string} (default header X-API-Key).
if (isJson)
{
if (jsonParsed &&
jsonFields!.TryGetValue("key", out var jsonKey) &&
!string.IsNullOrEmpty(jsonKey))
{
var header = jsonFields.TryGetValue("header", out var h) && !string.IsNullOrEmpty(h)
? h!
: "X-API-Key";
request.Headers.TryAddWithoutValidation(header, jsonKey);
}
else
{
// Malformed JSON, or JSON with no usable "key": send without a
// header and warn (the value is never logged). Never throw.
_logger.LogWarning(
"ApplyAuth: External system '{System}' AuthType 'apikey' AuthConfiguration is malformed JSON (expected {{\"header\"?:...,\"key\":...}}); request will be sent without an auth header.",
system.Name);
}
break;
}
// Legacy config format: "HeaderName:KeyValue" or just "KeyValue" (default header: X-API-Key)
var parts = system.AuthConfiguration.Split(':', 2);
if (parts.Length == 2)
{
@@ -500,7 +637,28 @@ public class ExternalSystemClient : IExternalSystemClient
break;
case "basic":
// Auth config format: "username:password"
// JSON form: {"username": string, "password": string}.
if (isJson)
{
if (jsonParsed &&
jsonFields!.TryGetValue("username", out var jsonUser) &&
!string.IsNullOrEmpty(jsonUser))
{
var password = jsonFields.TryGetValue("password", out var p) ? p ?? string.Empty : string.Empty;
var jsonEncoded = Convert.ToBase64String(
Encoding.UTF8.GetBytes($"{jsonUser}:{password}"));
request.Headers.Authorization = new AuthenticationHeaderValue("Basic", jsonEncoded);
}
else
{
_logger.LogWarning(
"ApplyAuth: External system '{System}' AuthType 'basic' AuthConfiguration is malformed JSON (expected {{\"username\":...,\"password\":...}}); request will be sent without an Authorization header.",
system.Name);
}
break;
}
// Legacy config format: "username:password"
var basicParts = system.AuthConfiguration.Split(':', 2);
if (basicParts.Length == 2)
{
@@ -532,6 +690,37 @@ public class ExternalSystemClient : IExternalSystemClient
}
}
/// <summary>
/// Parses a structured JSON <c>AuthConfiguration</c> into a case-insensitive
/// field map (e.g. <c>{"header":"X-Auth","key":"..."}</c> or
/// <c>{"username":"u","password":"p"}</c>). Returns <c>false</c> on malformed
/// JSON (<see cref="JsonException"/>) instead of throwing, so the caller can
/// fall through to the "send without header" warning path. Values are never
/// logged.
/// </summary>
private static bool TryParseJsonAuth(string config, out Dictionary<string, string?> fields)
{
try
{
fields = JsonSerializer.Deserialize<Dictionary<string, string?>>(
config,
new JsonSerializerOptions { PropertyNameCaseInsensitive = true })
?? new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
// Normalize to case-insensitive so "Header"/"header" both resolve.
if (!ReferenceEquals(fields.Comparer, StringComparer.OrdinalIgnoreCase))
{
fields = new Dictionary<string, string?>(fields, StringComparer.OrdinalIgnoreCase);
}
return true;
}
catch (JsonException)
{
fields = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
return false;
}
}
private async Task<(ExternalSystemDefinition? system, ExternalSystemMethod? method)> ResolveSystemAndMethodAsync(
string systemName,
string methodName,
@@ -10,4 +10,13 @@ public class ExternalSystemGatewayOptions
/// <summary>Maximum number of concurrent HTTP connections per external system.</summary>
public int MaxConcurrentConnectionsPerSystem { get; set; } = 10;
/// <summary>
/// Upper bound (bytes) on an outbound HTTP response body the gateway will
/// buffer into memory. A response whose <c>Content-Length</c> (when present)
/// or streamed length exceeds this cap fails permanently — retrying will not
/// shrink it, and buffering it into store-and-forward would defeat the cap.
/// Default 10 MiB.
/// </summary>
public long MaxResponseBodyBytes { get; set; } = 10 * 1024 * 1024;
}
+4 -4
View File
@@ -403,10 +403,10 @@ try
var apiRepo = scope.ServiceProvider.GetRequiredService<ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IInboundApiRepository>();
var executor = app.Services.GetRequiredService<ZB.MOM.WW.ScadaBridge.InboundAPI.InboundScriptExecutor>();
var methods = await apiRepo.GetAllApiMethodsAsync();
foreach (var method in methods)
{
executor.CompileAndRegister(method);
}
// Parallel: CompileAndRegister is safe for concurrent callers (ConcurrentDictionary
// caches, no shared mutable compile state); serial Roslyn compiles stretch the
// failover-recovery window at hundreds of methods (arch-review PLAN-06 Task 7).
Parallel.ForEach(methods, method => executor.CompileAndRegister(method));
}
await app.RunAsync();
@@ -51,6 +51,21 @@ public static class EndpointExtensions
/// </summary>
private const string UnauthorizedMessage = "Invalid or missing API key";
/// <summary>
/// C3: builds the canonical inbound-API error body — <c>{ "error": message, "code":
/// code }</c> — at the given HTTP status. Every error return in
/// <see cref="HandleInboundApiRequest"/> routes through this helper so the
/// machine-readable <c>code</c> field (the audience is external integrators) is
/// present and consistent across all failure branches. The <c>error</c> field is
/// unchanged, so callers that only read it are unaffected (additive field).
/// </summary>
/// <param name="code">The machine-readable error code (e.g. <c>UNAUTHORIZED</c>, <c>SITE_UNREACHABLE</c>).</param>
/// <param name="message">The human-readable, caller-safe error message.</param>
/// <param name="status">The HTTP status code for the response.</param>
/// <returns>A JSON result carrying the error + code at the given status.</returns>
private static IResult Error(string code, string message, int status) =>
Results.Json(new { error = message, code }, statusCode: status);
/// <summary>Registers the <c>POST /api/{methodName}</c> inbound API endpoint with the active-node gate and body-size filter applied.</summary>
/// <param name="endpoints">The route builder to add the endpoint to.</param>
/// <returns>The same <paramref name="endpoints"/> instance for chaining.</returns>
@@ -108,9 +123,7 @@ public static class EndpointExtensions
"Inbound API auth failure for method {Method}: {Failure} (status 401)",
methodName, verification.Failure);
return Results.Json(
new { error = UnauthorizedMessage },
statusCode: StatusCodes.Status401Unauthorized);
return Error("UNAUTHORIZED", UnauthorizedMessage, StatusCodes.Status401Unauthorized);
}
var identity = verification.Identity!;
@@ -150,9 +163,7 @@ public static class EndpointExtensions
"Inbound API authz failure for method {Method}: not approved (status 403)",
methodName);
return Results.Json(
new { error = NotApprovedMessage },
statusCode: StatusCodes.Status403Forbidden);
return Error("NOT_APPROVED", NotApprovedMessage, StatusCodes.Status403Forbidden);
}
// Telemetry follow-on: count this inbound request against the resolved,
@@ -168,6 +179,20 @@ public static class EndpointExtensions
httpContext.Items[AuditWriteMiddleware.AuditActorItemKey] =
identity.DisplayName;
// S9: a body sent with an explicit non-JSON Content-Type is a media-type
// mismatch, not malformed JSON — reject it with 415 rather than a misleading
// 400 "invalid JSON". A body with NO Content-Type header is still parsed
// leniently as JSON below (preserving existing callers). The sniff is
// ordinal-ignore-case so "application/JSON" etc. are treated as JSON.
if (httpContext.Request.ContentLength > 0
&& !string.IsNullOrEmpty(httpContext.Request.ContentType)
&& httpContext.Request.ContentType.Contains("json", StringComparison.OrdinalIgnoreCase) == false)
{
return Error(
"UNSUPPORTED_MEDIA_TYPE", "Content-Type must be application/json",
StatusCodes.Status415UnsupportedMediaType);
}
// Deserialize and validate parameters
JsonElement? body = null;
try
@@ -188,9 +213,7 @@ public static class EndpointExtensions
}
catch (JsonException)
{
return Results.Json(
new { error = "Invalid JSON in request body" },
statusCode: 400);
return Error("INVALID_JSON", "Invalid JSON in request body", StatusCodes.Status400BadRequest);
}
// Thread the JSON-Schema $ref resolution seam into parameter
@@ -209,9 +232,9 @@ public static class EndpointExtensions
var paramResult = ParameterValidator.Validate(body, method.ParameterDefinitions, resolveRef);
if (!paramResult.IsValid)
{
return Results.Json(
new { error = paramResult.ErrorMessage },
statusCode: 400);
return Error(
"VALIDATION_FAILED", paramResult.ErrorMessage ?? "Parameter validation failed",
StatusCodes.Status400BadRequest);
}
// Execute the method's script
@@ -250,9 +273,13 @@ public static class EndpointExtensions
"Inbound API script failure for method {Method}: {Error}",
methodName, scriptResult.ErrorMessage);
return Results.Json(
new { error = scriptResult.ErrorMessage ?? "Internal server error" },
statusCode: 500);
// C3: propagate the executor's machine-readable code (TIMEOUT,
// SITE_UNREACHABLE, SCRIPT_COMPILE_FAILED, …); an unclassified failure
// falls back to the SCRIPT_ERROR catch-all.
return Error(
scriptResult.ErrorCode ?? "SCRIPT_ERROR",
scriptResult.ErrorMessage ?? "Internal server error",
StatusCodes.Status500InternalServerError);
}
// Return the script result as JSON
@@ -24,17 +24,17 @@ namespace ZB.MOM.WW.ScadaBridge.InboundAPI;
public sealed class InboundApiEndpointFilter : IEndpointFilter
{
private readonly ILogger<InboundApiEndpointFilter> _logger;
private readonly InboundApiOptions _options;
private readonly IOptionsMonitor<InboundApiOptions> _options;
/// <summary>Initializes a new <see cref="InboundApiEndpointFilter"/> with the given logger and options.</summary>
/// <param name="logger">Logger for request rejection diagnostics.</param>
/// <param name="options">Inbound API options including the maximum request body size.</param>
/// <param name="options">Live-reloadable inbound API options, read per-request.</param>
public InboundApiEndpointFilter(
ILogger<InboundApiEndpointFilter> logger,
IOptions<InboundApiOptions> options)
IOptionsMonitor<InboundApiOptions> options)
{
_logger = logger;
_options = options.Value;
_options = options;
}
/// <summary>Applies active-node gating and request body size checks before delegating to the next filter or handler.</summary>
@@ -56,21 +56,21 @@ public sealed class InboundApiEndpointFilter : IEndpointFilter
_logger.LogWarning(
"Inbound API request rejected — this node is a standby (not the active central node)");
return Results.Json(
new { error = "Inbound API is only available on the active central node" },
new { error = "Inbound API is only available on the active central node", code = "STANDBY_NODE" },
statusCode: StatusCodes.Status503ServiceUnavailable);
}
// Cap the request body size. Reject an over-limit body up
// front via Content-Length; also lower the per-request max body size so a
// chunked/unknown-length stream is cut off by Kestrel as it is read.
var maxBytes = _options.MaxRequestBodyBytes;
var maxBytes = _options.CurrentValue.MaxRequestBodyBytes;
if (httpContext.Request.ContentLength is { } declaredLength && declaredLength > maxBytes)
{
_logger.LogWarning(
"Inbound API request rejected — body length {Length} exceeds limit {Limit}",
declaredLength, maxBytes);
return Results.Json(
new { error = "Request body too large" },
new { error = "Request body too large", code = "BODY_TOO_LARGE" },
statusCode: StatusCodes.Status413PayloadTooLarge);
}
@@ -8,6 +8,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.InboundApi;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
namespace ZB.MOM.WW.ScadaBridge.InboundAPI;
@@ -20,23 +21,35 @@ public class InboundScriptExecutor
{
private readonly ILogger<InboundScriptExecutor> _logger;
// A cached compiled handler, paired with the exact script text it was
// compiled from. The DB row is authoritative: on every request the cached
// Script is ordinal-compared against the freshly-fetched ApiMethod.Script; a
// mismatch means the cache is stale (a standby node compiled its own copy at
// startup, or the row was edited via direct SQL / bundle import) and the
// handler is recompiled in place. A null Script means "no revision info,
// serve as-is" — the RegisterHandler test seam, exempt from revision checks.
private sealed record CachedHandler(string? Script, Func<InboundScriptContext, Task<object?>> Handler);
// This executor is registered as a singleton and its handler cache
// is read and written from concurrent ASP.NET request threads. A plain Dictionary is
// not safe for concurrent read/write, so a ConcurrentDictionary is used throughout.
private readonly ConcurrentDictionary<string, Func<InboundScriptContext, Task<object?>>> _scriptHandlers = new();
private readonly ConcurrentDictionary<string, CachedHandler> _scriptHandlers = new();
// A script that fails to compile (or violates the trust model)
// is recorded here so it is compiled at most once. Without this, every subsequent
// is recorded here — keyed by method name, valued by the exact script text
// that failed — so it is compiled at most once. Without this, every subsequent
// request for a broken method re-runs the expensive Roslyn compilation — a CPU
// amplification vector since the inbound API has no rate limiting. The entry is
// cleared whenever the method is (re)compiled via CompileAndRegister.
// amplification vector since the inbound API has no rate limiting. A record only
// short-circuits when the CURRENT script text is the one that failed: a changed
// script always gets a fresh compile. The entry is cleared whenever the method
// is (re)compiled successfully.
//
// Bound the cache so a spam attack of unique method names cannot
// grow it without bound. Once the cap is reached new bad-method records are
// dropped — the cache is just a fast-fail optimisation; the per-request DB
// lookup remains the correctness path.
private const int KnownBadMethodsCap = 1000;
private readonly ConcurrentDictionary<string, byte> _knownBadMethods = new();
private readonly ConcurrentDictionary<string, string> _knownBadMethods = new();
/// <summary>
/// Diagnostic helper — returns the current size of the
@@ -46,24 +59,46 @@ public class InboundScriptExecutor
internal int KnownBadMethodCount => _knownBadMethods.Count;
/// <summary>
/// Records <paramref name="methodName"/> in the known-bad-methods
/// cache only if the cache has not reached <see cref="KnownBadMethodsCap"/>.
/// Once full, new records are dropped (paying the cheap recompile next time
/// rather than leaking memory under a unique-name flood). Existing entries are
/// not touched — they remain capped fast-fail records until cleared on a
/// successful (re)compile in <see cref="CompileAndRegister"/>.
/// Records <paramref name="methodName"/> → <paramref name="script"/> in the
/// known-bad-methods cache so the CURRENT failing script text short-circuits the
/// next request. An existing record for the method is overwritten so a newer bad
/// script replaces an older one (the fast-fail always tracks the latest failure).
/// A brand-new record is only added while the cache has not reached
/// <see cref="KnownBadMethodsCap"/>; once full, new method names are dropped
/// (paying the cheap recompile next time rather than leaking memory under a
/// unique-name flood). Records are cleared on a successful (re)compile.
/// </summary>
private void TryRecordBadMethod(string methodName)
private void TryRecordBadMethod(string methodName, string script)
{
// Overwrite an existing record (newer bad script wins) without counting
// against the cap; only a genuinely new method name is gated by the cap.
if (_knownBadMethods.ContainsKey(methodName))
{
_knownBadMethods[methodName] = script;
return;
}
if (_knownBadMethods.Count >= KnownBadMethodsCap)
return;
_knownBadMethods.TryAdd(methodName, 0);
_knownBadMethods.TryAdd(methodName, script);
}
private readonly IServiceProvider _serviceProvider;
// Count of inbound executions whose handler outlived its request (timed out or
// client-aborted) and is still running with its DI scope disposal deferred. A
// non-cooperative script that ignores the cancellation token keeps running after
// WaitAsync returns; the scope is disposed by the handler's own completion
// continuation, not the request finally, and this gauge makes the pile-up visible.
private long _abandonedExecutions;
/// <summary>
/// Diagnostic gauge — number of inbound executions currently orphaned (handler
/// still running after a timeout/abort, with DI-scope disposal deferred to the
/// handler's completion). Internal so the counter stays an implementation detail;
/// promote to public if a health surface needs it later.
/// </summary>
internal long AbandonedExecutionCount => Interlocked.Read(ref _abandonedExecutions);
/// <summary>
/// Initializes a new instance of the InboundScriptExecutor.
/// </summary>
@@ -76,24 +111,41 @@ public class InboundScriptExecutor
}
/// <summary>
/// Registers a compiled script handler for a method name.
/// Registers a compiled script handler for a method name. This is the script-less
/// test seam: the handler is stored with no associated script text, so it is
/// <b>exempt from the per-request revision check</b> and serves as-is regardless of
/// the fetched row's script. Production compiles always flow through
/// <see cref="CompileAndRegister(ApiMethod)"/>, which records the script text and is
/// revision-checked.
/// </summary>
/// <param name="methodName">The method name to register.</param>
/// <param name="handler">The compiled handler function.</param>
public void RegisterHandler(string methodName, Func<InboundScriptContext, Task<object?>> handler)
{
_scriptHandlers[methodName] = handler;
_scriptHandlers[methodName] = new CachedHandler(null, handler);
}
/// <summary>
/// Removes a compiled script handler for a method name.
/// Drops the compiled handler AND any known-bad record for the method. The
/// consumption seam for the script-artifact invalidation contract (bundle import,
/// out-of-band updates, failover activation). Safe to call for unknown names.
/// The per-request revision check in ExecuteAsync remains the correctness
/// fallback when an invalidation is missed.
/// </summary>
/// <param name="methodName">The method name to remove.</param>
public void RemoveHandler(string methodName)
/// <param name="methodName">The method name to invalidate.</param>
public void InvalidateMethod(string methodName)
{
_scriptHandlers.TryRemove(methodName, out _);
_knownBadMethods.TryRemove(methodName, out _);
}
/// <summary>
/// Removes a compiled script handler for a method name. Delegates to
/// <see cref="InvalidateMethod"/> (public API preserved).
/// </summary>
/// <param name="methodName">The method name to remove.</param>
public void RemoveHandler(string methodName) => InvalidateMethod(methodName);
/// <summary>
/// Compiles and registers a single API method script. Returns <c>false</c> if the
/// script is empty, fails Roslyn compilation, or violates the script trust model.
@@ -106,11 +158,14 @@ public class InboundScriptExecutor
/// Compiles and registers a single API method script, additionally reporting the
/// compilation diagnostics. Returns <c>false</c> together with the human-readable
/// error list if the script is empty, fails Roslyn compilation, or violates the
/// script trust model. On failure the <b>previously registered handler (if any) is
/// left in place</b> — a broken save never replaces a working delegate — so the
/// management Create/Update path can surface a non-fatal "saved but did not compile"
/// warning while the prior version keeps serving requests (see
/// <c>HandleCreateApiMethod</c>/<c>HandleUpdateApiMethod</c>).
/// script trust model. On failure this method does not overwrite any previously
/// registered handler, but the persisted DB row is authoritative: <see cref="ExecuteAsync"/>
/// revision-checks the cached handler against the freshly-fetched
/// <c>ApiMethod.Script</c> on every request, so once a broken script is saved the
/// method returns a compilation error consistently on every node rather than the
/// stale delegate silently serving. The management Create/Update path can therefore
/// surface a "saved but did not compile" warning that reflects real, consistent
/// behavior (see <c>HandleCreateApiMethod</c>/<c>HandleUpdateApiMethod</c>).
/// </summary>
/// <param name="method">The API method to compile and register.</param>
/// <param name="errors">On failure, the compilation/trust-model error messages; empty on success.</param>
@@ -125,7 +180,7 @@ public class InboundScriptExecutor
// through the capped TryRecordBadMethod helper so the cache
// cannot grow without bound under a flood of unique method names.
errors = compileErrors;
TryRecordBadMethod(method.Name);
TryRecordBadMethod(method.Name, method.Script ?? string.Empty);
return false;
}
@@ -133,12 +188,12 @@ public class InboundScriptExecutor
// failure record so a fixed script is no longer treated as bad.
errors = Array.Empty<string>();
_knownBadMethods.TryRemove(method.Name, out _);
return Register(method.Name, handler);
return Register(method, handler);
}
private bool Register(string methodName, Func<InboundScriptContext, Task<object?>> handler)
private bool Register(ApiMethod method, Func<InboundScriptContext, Task<object?>> handler)
{
_scriptHandlers[methodName] = handler;
_scriptHandlers[method.Name] = new CachedHandler(method.Script, handler);
return true;
}
@@ -266,6 +321,11 @@ public class InboundScriptExecutor
// no IServiceScopeFactory) is tolerated: the gateway is simply unavailable, so
// a script not using Database still runs, while one that does fails on use.
IServiceScope? scope = null;
// Hoisted out of the try so the finally can tell whether the handler task
// outlived the request. If it did (non-cooperative script after a timeout or
// client abort), disposing the scope underneath it would fault the still-running
// task with ObjectDisposedException — so the finally defers disposal instead.
Task<object?>? handlerTask = null;
try
{
scope = _serviceProvider.CreateScope();
@@ -308,29 +368,44 @@ public class InboundScriptExecutor
dbHelper,
cts.Token);
if (!_scriptHandlers.TryGetValue(method.Name, out var handler))
_scriptHandlers.TryGetValue(method.Name, out var cached);
// Revision check: a cached handler compiled from different script text is stale
// (standby-after-failover, direct SQL edit, bundle import). Null Script = test seam, exempt.
if (cached is { Script: not null }
&& !string.Equals(cached.Script, method.Script, StringComparison.Ordinal))
{
// A method already known to fail compilation must not
// be recompiled on every request — short-circuit before Roslyn runs.
if (_knownBadMethods.ContainsKey(method.Name))
return new InboundScriptResult(false, null, "Script compilation failed for this method");
_logger.LogInformation(
"API method {Method} script changed since it was cached; recompiling", method.Name);
cached = null;
}
// Lazy compile on first request (handles methods created after startup).
// Compile outside the cache so a failed compile is not stored, then add
// atomically so concurrent first-callers share a single handler instance.
if (cached == null)
{
// Fast-fail only when the CURRENT script text is the one already known bad.
if (_knownBadMethods.TryGetValue(method.Name, out var badScript)
&& string.Equals(badScript, method.Script, StringComparison.Ordinal))
return new InboundScriptResult(false, null, "Script compilation failed for this method", "SCRIPT_COMPILE_FAILED");
// Lazy compile (handles methods created after startup, and stale/known-bad
// recompiles). Compile outside the cache so a failed compile is not stored,
// then add so concurrent first-callers share a single handler instance.
var (compiled, _) = Compile(method);
if (compiled == null)
{
// Cache the failure so the next request short-circuits above.
// Routed through TryRecordBadMethod so the
// cache is bounded under a flood of unique method names.
TryRecordBadMethod(method.Name);
return new InboundScriptResult(false, null, "Script compilation failed for this method");
TryRecordBadMethod(method.Name, method.Script ?? string.Empty);
return new InboundScriptResult(false, null, "Script compilation failed for this method", "SCRIPT_COMPILE_FAILED");
}
handler = _scriptHandlers.GetOrAdd(method.Name, compiled);
_knownBadMethods.TryRemove(method.Name, out _);
cached = new CachedHandler(method.Script, compiled);
_scriptHandlers[method.Name] = cached; // last-write-wins; concurrent first-callers race benignly
}
var result = await handler(context).WaitAsync(cts.Token);
handlerTask = cached.Handler(context);
var result = await handlerTask.WaitAsync(cts.Token);
var resultJson = result != null
? JsonSerializer.Serialize(result)
@@ -373,23 +448,50 @@ public class InboundScriptExecutor
if (timeoutCts.IsCancellationRequested && !cancellationToken.IsCancellationRequested)
{
_logger.LogWarning("Script execution timed out for method {Method}", method.Name);
return new InboundScriptResult(false, null, "Script execution timed out");
return new InboundScriptResult(false, null, "Script execution timed out", "TIMEOUT");
}
_logger.LogDebug("Inbound API request for method {Method} cancelled by client", method.Name);
return new InboundScriptResult(false, null, "Request cancelled by client");
}
catch (SiteUnreachableException)
{
// C3: a routed Route.To(...).Call(...) reached a site that could not be
// contacted. Surface the machine-readable SITE_UNREACHABLE code (spec
// Component-InboundAPI failure body) instead of the generic SCRIPT_ERROR.
_logger.LogWarning("Routed site unreachable for method {Method}", method.Name);
return new InboundScriptResult(false, null, "Site unreachable", "SITE_UNREACHABLE");
}
catch (Exception ex)
{
_logger.LogError(ex, "Script execution failed for method {Method}", method.Name);
// Safe error message, no internal details
return new InboundScriptResult(false, null, "Internal script error");
return new InboundScriptResult(false, null, "Internal script error", "SCRIPT_ERROR");
}
finally
{
// Dispose the per-execution DI scope (if one was created) only after the
// script handler has finished running and its result been serialized.
scope?.Dispose();
if (handlerTask is null || handlerTask.IsCompleted)
{
scope?.Dispose();
}
else
{
// The handler outlived the request (non-cooperative script after a
// timeout/client abort). Do NOT dispose the scope underneath it —
// defer cleanup to the task's own completion, and count the orphan
// so repeated timeouts of a slow method are visible.
Interlocked.Increment(ref _abandonedExecutions);
ScadaBridgeTelemetry.RecordInboundAbandonedExecution(method.Name);
_logger.LogWarning(
"Abandoned inbound execution for method {Method} is still running; DI scope disposal deferred", method.Name);
var abandonedScope = scope;
_ = handlerTask.ContinueWith(t =>
{
_ = t.Exception; // observe the fault so it never surfaces as unobserved
abandonedScope?.Dispose();
Interlocked.Decrement(ref _abandonedExecutions);
}, TaskScheduler.Default);
}
}
}
}
@@ -443,7 +545,17 @@ public class InboundScriptContext
/// <summary>
/// Result of executing an inbound API script.
/// </summary>
/// <param name="Success">Whether the script executed successfully.</param>
/// <param name="ResultJson">The serialized script return value on success; null otherwise.</param>
/// <param name="ErrorMessage">A sanitized, caller-safe error message on failure; null on success.</param>
/// <param name="ErrorCode">
/// C3: an additive, machine-readable error code the endpoint surfaces in the failure
/// body's <c>code</c> field (e.g. <c>TIMEOUT</c>, <c>SITE_UNREACHABLE</c>,
/// <c>SCRIPT_COMPILE_FAILED</c>, <c>SCRIPT_ERROR</c>). Null when the executor does not
/// classify the failure; the endpoint then falls back to <c>SCRIPT_ERROR</c>.
/// </param>
public record InboundScriptResult(
bool Success,
string? ResultJson,
string? ErrorMessage);
string? ErrorMessage,
string? ErrorCode = null);
@@ -4,6 +4,22 @@ using ZB.MOM.WW.ScadaBridge.Commons.Types;
namespace ZB.MOM.WW.ScadaBridge.InboundAPI;
/// <summary>
/// C3: a routed call failed because the target site could not be reached (no contact /
/// unreachable). Derives from <see cref="InvalidOperationException"/> so existing
/// callers that catch the base type are unaffected, while
/// <see cref="InboundScriptExecutor"/> catches this more specific type to surface the
/// spec's <c>SITE_UNREACHABLE</c> error code instead of a generic <c>SCRIPT_ERROR</c>.
/// A routing failure whose message does <em>not</em> indicate unreachability stays a
/// plain <see cref="InvalidOperationException"/>.
/// </summary>
public sealed class SiteUnreachableException : InvalidOperationException
{
/// <summary>Initializes a new <see cref="SiteUnreachableException"/> with the given message.</summary>
/// <param name="message">The routing-level failure message that indicated the site was unreachable.</param>
public SiteUnreachableException(string message) : base(message) { }
}
/// <summary>
/// Route.To() helper for cross-site calls from inbound API scripts.
/// Resolves instance to site, routes via <see cref="IInstanceRouter"/>, blocks until
@@ -170,7 +186,7 @@ public class RouteTarget
if (!response.Success)
{
throw new InvalidOperationException(
throw RoutingFailure(
response.ErrorMessage ?? "Remote script call failed");
}
@@ -218,7 +234,7 @@ public class RouteTarget
if (!response.Success)
{
throw new InvalidOperationException(
throw RoutingFailure(
response.ErrorMessage ?? "Remote attribute read failed");
}
@@ -282,7 +298,7 @@ public class RouteTarget
if (!response.Success)
{
throw new InvalidOperationException(
throw RoutingFailure(
response.ErrorMessage ?? "Remote attribute wait failed");
}
@@ -333,7 +349,7 @@ public class RouteTarget
if (!response.Success)
{
throw new InvalidOperationException(
throw RoutingFailure(
response.ErrorMessage ?? "Remote attribute write failed");
}
}
@@ -350,10 +366,30 @@ public class RouteTarget
var siteId = await _instanceLocator.GetSiteIdForInstanceAsync(_instanceCode, cancellationToken);
if (siteId == null)
{
throw new InvalidOperationException(
// Route the site-resolution failure through the same classifier so a
// message indicating the site is unreachable surfaces as SITE_UNREACHABLE;
// a plain not-found / no-assigned-site stays a generic InvalidOperationException.
throw RoutingFailure(
$"Instance '{_instanceCode}' not found or has no assigned site");
}
return siteId;
}
/// <summary>
/// C3: classifies a routing-level failure message. When the message indicates the
/// site could not be reached (unreachable / no contact) the failure is a
/// <see cref="SiteUnreachableException"/> — so <see cref="InboundScriptExecutor"/>
/// can surface the spec's <c>SITE_UNREACHABLE</c> code — otherwise a plain
/// <see cref="InvalidOperationException"/>.
/// </summary>
private static InvalidOperationException RoutingFailure(string message) =>
IsUnreachable(message)
? new SiteUnreachableException(message)
: new InvalidOperationException(message);
private static bool IsUnreachable(string message) =>
message.Contains("unreachable", StringComparison.OrdinalIgnoreCase)
|| message.Contains("no contact", StringComparison.OrdinalIgnoreCase)
|| message.Contains("no-contact", StringComparison.OrdinalIgnoreCase);
}
@@ -1622,7 +1622,8 @@ public class ManagementActor : ReceiveActor
var repo = sp.GetRequiredService<IExternalSystemRepository>();
var def = new ExternalSystemDefinition(cmd.Name, cmd.EndpointUrl, cmd.AuthType)
{
AuthConfiguration = cmd.AuthConfiguration
AuthConfiguration = cmd.AuthConfiguration,
TimeoutSeconds = cmd.TimeoutSeconds ?? 0
};
await repo.AddExternalSystemAsync(def);
await repo.SaveChangesAsync();
@@ -1639,6 +1640,7 @@ public class ManagementActor : ReceiveActor
def.EndpointUrl = cmd.EndpointUrl;
def.AuthType = cmd.AuthType;
def.AuthConfiguration = cmd.AuthConfiguration;
if (cmd.TimeoutSeconds is { } ts) def.TimeoutSeconds = ts;
await repo.UpdateExternalSystemAsync(def);
await repo.SaveChangesAsync();
await AuditAsync(sp, user, "Update", "ExternalSystem", def.Id.ToString(), def.Name, def);
@@ -1818,6 +1820,8 @@ public class ManagementActor : ReceiveActor
c.MaxConcurrentConnections,
c.MaxRetries,
c.RetryDelay,
c.OAuth2Authority,
c.OAuth2Scope,
HasCredentials = !string.IsNullOrEmpty(c.Credentials),
};
@@ -1844,6 +1848,10 @@ public class ManagementActor : ReceiveActor
// existing values intact (non-breaking for callers that do not send them).
if (cmd.TlsMode is not null) config.TlsMode = cmd.TlsMode;
if (cmd.Credentials is not null) config.Credentials = cmd.Credentials;
// Preserve-if-null likewise for the OAuth2 token-endpoint overrides: an
// omitted authority/scope leaves the stored value (or its M365 default) intact.
if (cmd.OAuth2Authority is not null) config.OAuth2Authority = cmd.OAuth2Authority;
if (cmd.OAuth2Scope is not null) config.OAuth2Scope = cmd.OAuth2Scope;
await repo.UpdateSmtpConfigurationAsync(config);
await repo.SaveChangesAsync();
// Audit the credential-free shape — the *fact of* the change
@@ -1889,6 +1897,11 @@ public class ManagementActor : ReceiveActor
var configs = await repo.GetAllSmsConfigurationsAsync();
var config = configs.FirstOrDefault(c => c.Id == cmd.SmsConfigId)
?? throw new ManagementCommandException($"SmsConfiguration with ID {cmd.SmsConfigId} not found.");
// The AccountSid is interpolated un-escaped into the Twilio request URI
// (SmsNotificationDeliveryAdapter). Reject any value that is not a well-formed
// Twilio SID at save time, closing the URI-injection door before it is persisted.
if (!System.Text.RegularExpressions.Regex.IsMatch(cmd.AccountSid, "^AC[0-9a-fA-F]{32}$"))
throw new ManagementCommandException("Account SID must match Twilio's format: 'AC' followed by 32 hex characters.");
config.AccountSid = cmd.AccountSid;
config.FromNumber = cmd.FromNumber;
config.MessagingServiceSid = cmd.MessagingServiceSid;
@@ -2634,9 +2647,11 @@ public class ManagementActor : ReceiveActor
method.ReturnDefinition = cmd.ReturnDefinition;
await repo.UpdateApiMethodAsync(method);
await repo.SaveChangesAsync();
// On a compile failure the previously registered handler keeps serving — the
// saved-but-broken script does NOT go live. Surface that as a warning rather
// than a silent success (the change still persists either way).
// The persisted DB row is authoritative: on a compile failure the method returns
// a compilation error on every request (ExecuteAsync revision-checks the cached
// handler against the fresh row), so the saved-but-broken script does NOT keep a
// working version serving. Surface that as a warning rather than a silent success
// (the change still persists either way).
var compileWarning = TryCompileAndDescribe(sp, method, isUpdate: true);
await AuditAsync(sp, user, "Update", "ApiMethod", method.Id.ToString(), method.Name, method);
return ApiMethodResult(method, compileWarning);
@@ -2649,7 +2664,7 @@ public class ManagementActor : ReceiveActor
await repo.DeleteApiMethodAsync(cmd.ApiMethodId);
await repo.SaveChangesAsync();
if (method != null)
sp.GetService<InboundAPI.InboundScriptExecutor>()?.RemoveHandler(method.Name);
sp.GetService<InboundAPI.InboundScriptExecutor>()?.InvalidateMethod(method.Name);
await AuditAsync(sp, user, "Delete", "ApiMethod", cmd.ApiMethodId.ToString(), method?.Name ?? cmd.ApiMethodId.ToString(), null);
return true;
}
@@ -2657,8 +2672,10 @@ public class ManagementActor : ReceiveActor
/// <summary>
/// Recompiles and re-registers an inbound API method's script after a Create/Update,
/// returning a non-null human-readable warning when the script did NOT compile. The
/// save is always persisted; on compile failure the previously registered handler (if
/// any) keeps serving requests — the broken script never replaces a working one (see
/// save is always persisted; the persisted DB row is authoritative, so on compile
/// failure the method returns a compilation error on every request (ExecuteAsync
/// revision-checks the cached handler against the fresh row) rather than a previously
/// registered handler silently continuing to serve (see
/// <c>InboundScriptExecutor.CompileAndRegister</c>). Returns <c>null</c> when the
/// script compiled, or when no executor is registered (e.g. a non-inbound host or a
/// test double).
@@ -2673,7 +2690,7 @@ public class ManagementActor : ReceiveActor
var detail = string.Join("; ", errors);
return isUpdate
? $"Script saved but failed to compile; the previously registered version (if any) remains active and the new script is NOT live. Compilation errors: {detail}"
? $"Script saved but failed to compile; the method will return a compilation error on every request until a compiling version is saved. Compilation errors: {detail}"
: $"Script saved but failed to compile; the method will return a compilation error until a compiling version is saved. Compilation errors: {detail}";
}
@@ -73,12 +73,22 @@ public sealed class EmailNotificationDeliveryAdapter : INotificationDeliveryAdap
}
var smtpConfigs = await _repository.GetAllSmtpConfigurationsAsync(cancellationToken);
var smtpConfig = smtpConfigs.FirstOrDefault();
// Deterministic pick: the lowest-Id row wins regardless of repository order,
// so every node/attempt agrees on the same configuration (arch-review S7).
var smtpConfig = smtpConfigs.OrderBy(c => c.Id).FirstOrDefault();
if (smtpConfig == null)
{
return DeliveryOutcome.Permanent("No SMTP configuration available");
}
if (smtpConfigs.Count > 1)
{
_logger.LogWarning(
"Multiple SMTP configurations exist; using the lowest-Id row ({Id}). "
+ "Enforce a single row or delete extras.",
smtpConfig.Id);
}
// An unknown TLS mode is a configuration error that retrying cannot fix —
// surface it as a permanent failure (SMTP TLS validation policy).
SmtpTlsMode tlsMode;
@@ -185,7 +195,11 @@ public sealed class EmailNotificationDeliveryAdapter : INotificationDeliveryAdap
if (config.AuthType.Equals("oauth2", StringComparison.OrdinalIgnoreCase)
&& _tokenService != null && credentials != null)
{
credentials = await _tokenService.GetTokenAsync(credentials, cancellationToken);
credentials = await _tokenService.GetTokenAsync(
credentials,
config.OAuth2Authority,
config.OAuth2Scope,
cancellationToken);
}
// OAuth2 XOAUTH2 requires the user identity (FromAddress)
@@ -155,17 +155,22 @@ public sealed class SmsNotificationDeliveryAdapter : INotificationDeliveryAdapte
httpClient, requestUri, authHeader, smsConfig, hasMessagingService,
phone, body, timeoutSeconds, redactSecret, cancellationToken);
switch (attempt.Class)
if (attempt.Class == SmsErrorClass.Transient)
{
case SmsErrorClass.Transient:
firstTransientError ??= attempt.Detail;
break;
case SmsErrorClass.Permanent:
permanentlyFailed.Add(phone);
break;
default:
accepted.Add(phone);
break;
// Every recipient accepted after the first transient is a guaranteed
// duplicate on retry (the whole notification re-sends); stopping here
// also bounds a black-holed endpoint to one timeout per sweep.
firstTransientError = attempt.Detail;
break;
}
if (attempt.Class == SmsErrorClass.Permanent)
{
permanentlyFailed.Add(phone);
}
else
{
accepted.Add(phone);
}
}
@@ -284,6 +289,13 @@ public sealed class SmsNotificationDeliveryAdapter : INotificationDeliveryAdapte
/// any transient → Transient (whole notification retries); else any
/// accepted → Success (the permanently-failed numbers are noted, NOT parked); else
/// (all permanent / zero accepted) → Permanent.
/// <para>
/// The caller's recipient loop stops at the FIRST transient rather than attempting
/// every recipient: every recipient accepted after the first transient is a
/// guaranteed duplicate on retry (the whole notification re-sends), so
/// <paramref name="accepted"/> and <paramref name="permanentlyFailed"/> only ever
/// reflect recipients attempted before that first transient.
/// </para>
/// </summary>
private DeliveryOutcome RollUp(
string listName,
@@ -316,7 +316,7 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
return;
}
var (maxRetries, retryDelay) = await ResolveRetryPolicyAsync(notificationRepository);
var policies = await ResolveRetryPoliciesAsync(notificationRepository);
// Deliver the claimed batch with bounded parallelism. Delivery is I/O-bound
// (multi-second SMTP/Twilio round trips), so a strictly sequential sweep caps
@@ -333,7 +333,7 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
foreach (var notification in due)
{
deliveries.Add(DeliverGatedAsync(
notification, now, maxRetries, retryDelay, adapters, deliveryGate, cancellationToken));
notification, now, policies, adapters, deliveryGate, cancellationToken));
}
// Each DeliverGatedAsync isolates its own faults, so WhenAll never throws.
@@ -358,8 +358,7 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
private async Task DeliverGatedAsync(
Notification notification,
DateTimeOffset now,
int maxRetries,
TimeSpan retryDelay,
RetryPolicies policies,
IReadOnlyDictionary<NotificationType, INotificationDeliveryAdapter> adapters,
SemaphoreSlim gate,
CancellationToken cancellationToken)
@@ -388,7 +387,7 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
using var scope = _serviceProvider.CreateScope();
var outboxRepository = scope.ServiceProvider.GetRequiredService<INotificationOutboxRepository>();
await DeliverOneAsync(
notification, now, maxRetries, retryDelay, outboxRepository, adapters, cancellationToken)
notification, now, policies, outboxRepository, adapters, cancellationToken)
.ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
@@ -409,52 +408,117 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
}
/// <summary>
/// Resolves the retry policy from the first SMTP configuration row. When no SMTP
/// configuration exists, falls back to a conservative default — delivery itself will
/// permanently fail in that case, so the policy only acts as a guard.
/// A single channel's transient-failure retry policy: the max attempt count before a
/// row parks and the fixed delay between attempts. Resolved once per sweep and selected
/// per notification by <see cref="NotificationType"/> in <see cref="DeliverOneAsync"/>.
/// </summary>
/// <remarks>
/// A non-positive <see cref="SmtpConfiguration.MaxRetries"/> (zero or negative)
/// would otherwise satisfy <c>RetryCount >= maxRetries</c> on the very first transient
/// failure and park the row without a single retry — silently halving the outbox's
/// delivery guarantees. The same applies to a non-positive <c>RetryDelay</c>, which
/// would burn-loop the dispatcher. Both values are clamped to the fallback constants
/// with a Warning so an operator can spot the misconfiguration in logs.
/// </remarks>
private async Task<(int MaxRetries, TimeSpan RetryDelay)> ResolveRetryPolicyAsync(
private readonly record struct RetryPolicy(int MaxRetries, TimeSpan RetryDelay);
/// <summary>
/// The per-type retry policies resolved once per dispatch sweep: <see cref="SmtpPolicy"/>
/// drives Email (and every non-SMS type), <see cref="SmsPolicy"/> drives SMS.
/// </summary>
private readonly record struct RetryPolicies(RetryPolicy SmtpPolicy, RetryPolicy SmsPolicy);
/// <summary>
/// Resolves the per-type retry policies for the sweep: the SMTP policy from the first
/// SMTP configuration row (drives Email and every non-SMS type), and the SMS policy from
/// the <see cref="SmsConfiguration"/> row — falling back to the SMTP policy when no SMS
/// configuration exists. Resolved once and reused for the whole batch;
/// <see cref="DeliverOneAsync"/> selects the right policy by
/// <see cref="Notification.Type"/>.
/// </summary>
private async Task<RetryPolicies> ResolveRetryPoliciesAsync(
INotificationRepository notificationRepository)
{
var smtpPolicy = await ResolveSmtpPolicyAsync(notificationRepository);
var smsPolicy = await ResolveSmsPolicyAsync(notificationRepository, smtpPolicy);
return new RetryPolicies(smtpPolicy, smsPolicy);
}
/// <summary>
/// Resolves the SMTP-derived retry policy from the lowest-Id SMTP configuration row. When no
/// SMTP configuration exists, falls back to the conservative default constants — delivery
/// itself will permanently fail in that case, so the policy only acts as a guard.
/// </summary>
private async Task<RetryPolicy> ResolveSmtpPolicyAsync(
INotificationRepository notificationRepository)
{
var configurations = await notificationRepository.GetAllSmtpConfigurationsAsync();
var configuration = configurations.Count > 0 ? configurations[0] : null;
// Deterministic pick: the lowest-Id row wins regardless of repository order, so the
// retry policy resolves the same way the delivery adapter selects its config (S7).
var configuration = configurations.OrderBy(c => c.Id).FirstOrDefault();
if (configuration is null)
{
return (FallbackMaxRetries, FallbackRetryDelay);
return new RetryPolicy(FallbackMaxRetries, FallbackRetryDelay);
}
var maxRetries = configuration.MaxRetries;
var retryDelay = configuration.RetryDelay;
if (configurations.Count > 1)
{
_logger.LogWarning(
"Multiple SMTP configurations exist; using the lowest-Id row ({Id}). "
+ "Enforce a single row or delete extras.",
configuration.Id);
}
return ClampRetryPolicy(
nameof(SmtpConfiguration), configuration.MaxRetries, configuration.RetryDelay);
}
/// <summary>
/// Resolves the SMS-specific retry policy from the <see cref="SmsConfiguration"/> row,
/// honoring its own <see cref="SmsConfiguration.MaxRetries"/>/<see cref="SmsConfiguration.RetryDelay"/>.
/// When no SMS configuration row exists the SMTP policy (<paramref name="smtpFallback"/>)
/// is reused so SMS delivery keeps the historical shared-SMTP behaviour until an operator
/// configures SMS.
/// </summary>
private async Task<RetryPolicy> ResolveSmsPolicyAsync(
INotificationRepository notificationRepository, RetryPolicy smtpFallback)
{
var configuration = await notificationRepository.GetSmsConfigurationAsync();
if (configuration is null)
{
return smtpFallback;
}
return ClampRetryPolicy(
nameof(SmsConfiguration), configuration.MaxRetries, configuration.RetryDelay);
}
/// <summary>
/// Clamps a configured retry policy's non-positive values to the fallback constants,
/// logging a Warning (parameterized by <paramref name="configName"/> so an operator can
/// tell which configuration is misconfigured).
/// </summary>
/// <remarks>
/// A non-positive <c>MaxRetries</c> (zero or negative) would otherwise satisfy
/// <c>RetryCount >= maxRetries</c> on the very first transient failure and park the row
/// without a single retry — silently halving the outbox's delivery guarantees. The same
/// applies to a non-positive <c>RetryDelay</c>, which would burn-loop the dispatcher.
/// </remarks>
private RetryPolicy ClampRetryPolicy(string configName, int maxRetries, TimeSpan retryDelay)
{
if (maxRetries <= 0)
{
_logger.LogWarning(
"SmtpConfiguration.MaxRetries={ConfiguredMaxRetries} is non-positive; " +
"{ConfigName}.MaxRetries={ConfiguredMaxRetries} is non-positive; " +
"clamping to FallbackMaxRetries={FallbackMaxRetries} so transient failures " +
"actually retry before parking.",
maxRetries, FallbackMaxRetries);
configName, maxRetries, FallbackMaxRetries);
maxRetries = FallbackMaxRetries;
}
if (retryDelay <= TimeSpan.Zero)
{
_logger.LogWarning(
"SmtpConfiguration.RetryDelay={ConfiguredRetryDelay} is non-positive; " +
"{ConfigName}.RetryDelay={ConfiguredRetryDelay} is non-positive; " +
"clamping to FallbackRetryDelay={FallbackRetryDelay} so the dispatcher does " +
"not burn-loop on transient failures.",
retryDelay, FallbackRetryDelay);
configName, retryDelay, FallbackRetryDelay);
retryDelay = FallbackRetryDelay;
}
return (maxRetries, retryDelay);
return new RetryPolicy(maxRetries, retryDelay);
}
/// <summary>
@@ -519,12 +583,17 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
private async Task DeliverOneAsync(
Notification notification,
DateTimeOffset now,
int maxRetries,
TimeSpan retryDelay,
RetryPolicies policies,
INotificationOutboxRepository outboxRepository,
IReadOnlyDictionary<NotificationType, INotificationDeliveryAdapter> adapters,
CancellationToken cancellationToken)
{
// Select the retry policy by delivery channel: SMS honors SmsConfiguration's own
// MaxRetries/RetryDelay; every other type (Email today) uses the SMTP policy.
var (maxRetries, retryDelay) = notification.Type == NotificationType.Sms
? policies.SmsPolicy
: policies.SmtpPolicy;
if (!adapters.TryGetValue(notification.Type, out var adapter))
{
// Missing-adapter park: from the dispatcher's perspective this is an
@@ -38,25 +38,40 @@ public class OAuth2TokenService
/// Credentials format: "tenantId:clientId:clientSecret"
/// </summary>
/// <param name="credentials">Colon-separated string in the form <c>tenantId:clientId:clientSecret</c>.</param>
/// <param name="authority">
/// Optional token-endpoint authority template (may contain a <c>{tenant}</c> placeholder);
/// null preserves the Microsoft 365 default endpoint.
/// </param>
/// <param name="scope">Optional OAuth2 scope; null preserves the Microsoft 365 default scope.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>A valid OAuth2 access token string.</returns>
public async Task<string> GetTokenAsync(string credentials, CancellationToken cancellationToken = default)
public async Task<string> GetTokenAsync(
string credentials,
string? authority = null,
string? scope = null,
CancellationToken cancellationToken = default)
{
var key = CredentialKey(credentials);
// Cache key incorporates authority + scope so two configurations with the same
// credentials but different endpoints/scopes never share a token.
var key = CredentialKey($"{credentials}|{authority}|{scope}");
var entry = _cache.GetOrAdd(key, _ => new CacheEntry());
if (entry.Token != null && DateTimeOffset.UtcNow < entry.Expiry)
// Read the token/expiry pair through a single volatile reference so a lock-free
// reader can never observe a torn token/expiry combination (arch-review S8).
var current = entry.Current;
if (current != null && DateTimeOffset.UtcNow < current.Expiry)
{
return entry.Token;
return current.Token;
}
await entry.Lock.WaitAsync(cancellationToken);
try
{
// Double-check after acquiring the per-credential lock.
if (entry.Token != null && DateTimeOffset.UtcNow < entry.Expiry)
current = entry.Current;
if (current != null && DateTimeOffset.UtcNow < current.Expiry)
{
return entry.Token;
return current.Token;
}
var parts = credentials.Split(':', 3);
@@ -70,14 +85,22 @@ public class OAuth2TokenService
var clientSecret = parts[2];
var client = _httpClientFactory.CreateClient("OAuth2");
var tokenUrl = $"https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token";
// Authority: null → today's hardcoded M365 endpoint; otherwise the stored
// template with an optional {tenant} placeholder resolved from the credential.
var tokenUrl = authority == null
? $"https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token"
: authority.Replace("{tenant}", tenantId);
// Scope: null → the M365 default.
var tokenScope = scope ?? "https://outlook.office365.com/.default";
var form = new FormUrlEncodedContent(new Dictionary<string, string>
{
["grant_type"] = "client_credentials",
["client_id"] = clientId,
["client_secret"] = clientSecret,
["scope"] = "https://outlook.office365.com/.default"
["scope"] = tokenScope
});
var response = await client.PostAsync(tokenUrl, form, cancellationToken);
@@ -90,8 +113,10 @@ public class OAuth2TokenService
?? throw new InvalidOperationException("No access_token in OAuth2 response");
var expiresIn = doc.RootElement.GetProperty("expires_in").GetInt32();
entry.Token = token;
entry.Expiry = DateTimeOffset.UtcNow.AddSeconds(expiresIn - 60); // Refresh 60s before expiry
// Publish the token/expiry pair as a single atomic reference assignment
// (Refresh 60s before expiry).
entry.Current = new TokenValue(
token, DateTimeOffset.UtcNow.AddSeconds(expiresIn - 60));
// The token endpoint identity is logged by tenant only — never
// the client secret or the access token itself.
@@ -115,10 +140,14 @@ public class OAuth2TokenService
return Convert.ToHexString(hash);
}
/// <summary>An atomically-published token/expiry pair — one reference, never torn.</summary>
private sealed record TokenValue(string Token, DateTimeOffset Expiry);
private sealed class CacheEntry
{
public string? Token;
public DateTimeOffset Expiry = DateTimeOffset.MinValue;
// Written only under Lock; read lock-free via the volatile reference so a reader
// always sees a self-consistent token/expiry pair (or null for "no token yet").
public volatile TokenValue? Current;
public readonly SemaphoreSlim Lock = new(1, 1);
}
}
@@ -1885,7 +1885,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
foreach (var es in command.ExternalSystems)
{
await _storage.StoreExternalSystemAsync(es.Name, es.EndpointUrl,
es.AuthType, es.AuthConfiguration, es.MethodDefinitionsJson);
es.AuthType, es.AuthConfiguration, es.MethodDefinitionsJson, es.TimeoutSeconds);
}
}
@@ -349,7 +349,7 @@ public class SiteReplicationActor : ReceiveActor
if (command.ExternalSystems != null)
foreach (var es in command.ExternalSystems)
await _storage.StoreExternalSystemAsync(es.Name, es.EndpointUrl, es.AuthType, es.AuthConfiguration, es.MethodDefinitionsJson);
await _storage.StoreExternalSystemAsync(es.Name, es.EndpointUrl, es.AuthType, es.AuthConfiguration, es.MethodDefinitionsJson, es.TimeoutSeconds);
if (command.DatabaseConnections != null)
foreach (var db in command.DatabaseConnections)
@@ -174,6 +174,10 @@ public class SiteStorageService
// Native-alarm display metadata (UA4) — restored on rehydration so a persisted condition
// renders fully (type/category/message/values) before the first source snapshot arrives.
await TryAddColumnAsync(connection, "native_alarm_state", "metadata_json", "TEXT");
// Per-external-system call timeout (ExternalSystemGateway Timeout) — carried through the
// artifact pipeline so site-side calls honor it; 0 = use the site default.
await TryAddColumnAsync(connection, "external_systems", "timeout_seconds", "INTEGER NOT NULL DEFAULT 0");
}
private async Task TryAddColumnAsync(SqliteConnection connection, string table, string column, string type)
@@ -716,22 +720,25 @@ public class SiteStorageService
/// <param name="authType">The authentication type (e.g., 'ApiKey', 'BasicAuth').</param>
/// <param name="authConfig">Authentication configuration JSON, if applicable.</param>
/// <param name="methodDefs">JSON representation of available method definitions, if any.</param>
/// <param name="timeoutSeconds">Per-system call timeout in seconds (0 = use the site default).</param>
/// <returns>A task that completes when the external system definition has been stored or updated.</returns>
public async Task StoreExternalSystemAsync(
string name, string endpointUrl, string authType, string? authConfig, string? methodDefs)
string name, string endpointUrl, string authType, string? authConfig, string? methodDefs,
int timeoutSeconds = 0)
{
await using var connection = new SqliteConnection(_connectionString);
await connection.OpenAsync();
await using var command = connection.CreateCommand();
command.CommandText = @"
INSERT INTO external_systems (name, endpoint_url, auth_type, auth_configuration, method_definitions, updated_at)
VALUES (@name, @url, @authType, @authConfig, @methodDefs, @updatedAt)
INSERT INTO external_systems (name, endpoint_url, auth_type, auth_configuration, method_definitions, timeout_seconds, updated_at)
VALUES (@name, @url, @authType, @authConfig, @methodDefs, @timeoutSeconds, @updatedAt)
ON CONFLICT(name) DO UPDATE SET
endpoint_url = excluded.endpoint_url,
auth_type = excluded.auth_type,
auth_configuration = excluded.auth_configuration,
method_definitions = excluded.method_definitions,
timeout_seconds = excluded.timeout_seconds,
updated_at = excluded.updated_at";
command.Parameters.AddWithValue("@name", name);
@@ -739,6 +746,7 @@ public class SiteStorageService
command.Parameters.AddWithValue("@authType", authType);
command.Parameters.AddWithValue("@authConfig", (object?)authConfig ?? DBNull.Value);
command.Parameters.AddWithValue("@methodDefs", (object?)methodDefs ?? DBNull.Value);
command.Parameters.AddWithValue("@timeoutSeconds", timeoutSeconds);
command.Parameters.AddWithValue("@updatedAt", DateTimeOffset.UtcNow.ToString("O"));
await command.ExecuteNonQueryAsync();
@@ -36,7 +36,7 @@ public class SiteExternalSystemRepository : IExternalSystemRepository
await using var command = connection.CreateCommand();
command.CommandText = @"
SELECT name, endpoint_url, auth_type, auth_configuration
SELECT name, endpoint_url, auth_type, auth_configuration, timeout_seconds
FROM external_systems";
var results = new List<ExternalSystemDefinition>();
@@ -68,7 +68,7 @@ public class SiteExternalSystemRepository : IExternalSystemRepository
await using var command = connection.CreateCommand();
command.CommandText = @"
SELECT name, endpoint_url, auth_type, auth_configuration
SELECT name, endpoint_url, auth_type, auth_configuration, timeout_seconds
FROM external_systems
WHERE name = @name";
command.Parameters.AddWithValue("@name", name);
@@ -263,7 +263,8 @@ public class SiteExternalSystemRepository : IExternalSystemRepository
authType: reader.GetString(2))
{
Id = GenerateSyntheticId(name),
AuthConfiguration = reader.IsDBNull(3) ? null : reader.GetString(3)
AuthConfiguration = reader.IsDBNull(3) ? null : reader.GetString(3),
TimeoutSeconds = reader.IsDBNull(4) ? 0 : reader.GetInt32(4)
};
}
@@ -0,0 +1,79 @@
using System.CommandLine;
using ZB.MOM.WW.ScadaBridge.CLI.Commands;
namespace ZB.MOM.WW.ScadaBridge.CLI.Tests.Commands;
/// <summary>
/// PLAN-06 Task 11: <c>external-system create</c> and <c>external-system update</c>
/// must expose an optional <c>--timeout-seconds</c> (<see cref="int"/>?) flag that
/// carries the per-system ESG HTTP timeout. Omitting it preserves the existing value
/// (update) / uses the gateway default (create).
/// </summary>
public class ExternalSystemTimeoutOptionTests
{
private static readonly Option<string> Url = new("--url") { Recursive = true };
private static readonly Option<string> Format = new("--format") { Recursive = true };
private static readonly Option<string> Username = new("--username") { Recursive = true };
private static readonly Option<string> Password = new("--password") { Recursive = true };
private static Command Group() => ExternalSystemCommands.Build(Url, Format, Username, Password);
private static Option? FindOption(Command command, string name)
=> command.Options.SingleOrDefault(o => o.Name == name);
[Fact]
public void Create_HasOptionalTimeoutSecondsOption()
{
var create = Group().Subcommands.Single(c => c.Name == "create");
var option = FindOption(create, "--timeout-seconds");
Assert.NotNull(option);
Assert.False(option!.Required);
}
[Fact]
public void Update_HasOptionalTimeoutSecondsOption()
{
var update = Group().Subcommands.Single(c => c.Name == "update");
var option = FindOption(update, "--timeout-seconds");
Assert.NotNull(option);
Assert.False(option!.Required);
}
[Fact]
public void Create_ParsesTimeoutSecondsValue()
{
var parse = Group().Parse(new[]
{
"create", "--name", "Sys", "--endpoint-url", "http://x",
"--auth-type", "none", "--timeout-seconds", "15",
});
Assert.Empty(parse.Errors);
Assert.Equal(15, parse.GetValue<int?>("--timeout-seconds"));
}
[Fact]
public void Update_ParsesTimeoutSecondsValue()
{
var parse = Group().Parse(new[]
{
"update", "--id", "7", "--name", "Sys", "--endpoint-url", "http://x",
"--auth-type", "none", "--timeout-seconds", "20",
});
Assert.Empty(parse.Errors);
Assert.Equal(20, parse.GetValue<int?>("--timeout-seconds"));
}
[Fact]
public void Create_OmittedTimeoutSeconds_ParsesWithoutError()
{
var parse = Group().Parse(new[]
{
"create", "--name", "Sys", "--endpoint-url", "http://x", "--auth-type", "none",
});
Assert.Empty(parse.Errors);
Assert.Null(parse.GetValue<int?>("--timeout-seconds"));
}
}
@@ -1,3 +1,4 @@
using System.Linq;
using System.Security.Claims;
using Bunit;
using Microsoft.AspNetCore.Components;
@@ -67,4 +68,65 @@ public class ExternalSystemFormAuditDrillinTests : BunitContext
Assert.Empty(cut.FindAll("a[data-test=\"audit-link\"]"));
});
}
[Fact]
public void Form_RendersTimeoutSecondsInput_WithGatewayDefaultHelperText()
{
// PLAN-06 Task 12: the per-system TimeoutSeconds column (Task 8) needs a
// Central UI form field so Design users can actually set it.
var cut = Render<ExternalSystemForm>();
cut.WaitForAssertion(() =>
{
Assert.Contains("Timeout (seconds)", cut.Markup);
Assert.Contains("0 = use gateway default", cut.Markup);
});
}
[Fact]
public void SavingNewSystem_WithTimeoutSeconds_PassesTimeoutSecondsToRepository()
{
var cut = Render<ExternalSystemForm>();
// Re-query between each Change(): two-way binding re-renders the form and
// invalidates previously found element references.
cut.FindAll("input[type=text]")[0].Change("ERP-Beta"); // Name
cut.FindAll("input[type=text]")[1].Change("https://erp-beta.example.test"); // Endpoint URL
cut.FindAll("input[type=number]")[2].Change("20"); // Timeout (seconds)
cut.FindAll("button").First(b => b.TextContent.Contains("Save")).Click();
cut.WaitForAssertion(() =>
{
_repo.Received().AddExternalSystemAsync(
Arg.Is<ExternalSystemDefinition>(e => e.TimeoutSeconds == 20));
_repo.Received().SaveChangesAsync();
});
}
[Fact]
public void SavingEditedSystem_UpdatesTimeoutSecondsOnExisting()
{
var existing = new ExternalSystemDefinition("ERP-Delta", "https://erp-delta.example.test", "ApiKey")
{
Id = 11,
TimeoutSeconds = 10,
};
_repo.GetExternalSystemByIdAsync(11, Arg.Any<CancellationToken>()).Returns(existing);
var cut = Render<ExternalSystemForm>(p => p.Add(c => c.Id, 11));
cut.WaitForState(() => cut.FindAll("input[type=number]").Count >= 3);
// The Timeout field must be pre-filled from the existing entity on load.
Assert.Contains(cut.FindAll("input[type=number]"), i => i.GetAttribute("value") == "10");
cut.FindAll("input[type=number]")[2].Change("60"); // Timeout (seconds)
cut.FindAll("button").First(b => b.TextContent.Contains("Save")).Click();
cut.WaitForAssertion(() =>
{
Assert.Equal(60, existing.TimeoutSeconds);
_repo.Received().UpdateExternalSystemAsync(existing);
});
}
}
@@ -32,7 +32,7 @@ public class SmsConfigurationPageTests : BunitContext
private const string SecretToken = "super-secret-auth-token";
private static SmsConfiguration Sample() =>
new("ACtest_account_sid", "+15551234567")
new("ACcccccccccccccccccccccccccccccccc", "+15551234567")
{
Id = 1,
MessagingServiceSid = "MGtest_messaging_service",
@@ -63,7 +63,7 @@ public class SmsConfigurationPageTests : BunitContext
cut.WaitForAssertion(() =>
{
// Config row fields render.
Assert.Contains("ACtest_account_sid", cut.Markup);
Assert.Contains("ACcccccccccccccccccccccccccccccccc", cut.Markup);
Assert.Contains("+15551234567", cut.Markup);
Assert.Contains("MGtest_messaging_service", cut.Markup);
// Auth Token shows a presence indicator only — never the value.
@@ -81,7 +81,7 @@ public class SmsConfigurationPageTests : BunitContext
WireAuth();
var cut = Render<SmsConfigurationPage>();
cut.WaitForState(() => cut.Markup.Contains("ACtest_account_sid"));
cut.WaitForState(() => cut.Markup.Contains("ACcccccccccccccccccccccccccccccccc"));
cut.FindAll("button").First(b => b.TextContent.Contains("Edit")).Click();
@@ -109,7 +109,7 @@ public class SmsConfigurationPageTests : BunitContext
// Re-query between each Change(): two-way binding re-renders the form and
// invalidates previously found element references.
cut.FindAll("input[type=text]")[0].Change("ACnew_account"); // Account SID
cut.FindAll("input[type=text]")[0].Change("ACdddddddddddddddddddddddddddddddd"); // Account SID
cut.FindAll("input[type=text]")[1].Change("+15559876543"); // From Number
cut.FindAll("input[type=password]").First().Change("new-token");
@@ -119,7 +119,7 @@ public class SmsConfigurationPageTests : BunitContext
{
repo.Received().AddSmsConfigurationAsync(
Arg.Is<SmsConfiguration>(c =>
c.AccountSid == "ACnew_account" &&
c.AccountSid == "ACdddddddddddddddddddddddddddddddd" &&
c.FromNumber == "+15559876543" &&
c.AuthToken == "new-token"));
repo.Received().SaveChangesAsync();
@@ -141,7 +141,7 @@ public class SmsConfigurationPageTests : BunitContext
cut.FindAll("button").First(b => b.TextContent.Contains("Add SMS configuration")).Click();
// Account SID + Messaging Service SID, leaving From Number (index 1) blank.
cut.FindAll("input[type=text]")[0].Change("ACmsg_account"); // Account SID
cut.FindAll("input[type=text]")[0].Change("ACeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"); // Account SID
cut.FindAll("input[type=text]")[2].Change("MGmessaging_service"); // Messaging Service SID
cut.FindAll("input[type=password]").First().Change("new-token");
@@ -151,7 +151,7 @@ public class SmsConfigurationPageTests : BunitContext
{
repo.Received().AddSmsConfigurationAsync(
Arg.Is<SmsConfiguration>(c =>
c.AccountSid == "ACmsg_account" &&
c.AccountSid == "ACeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" &&
c.FromNumber == null &&
c.MessagingServiceSid == "MGmessaging_service" &&
c.AuthToken == "new-token"));
@@ -168,7 +168,7 @@ public class SmsConfigurationPageTests : BunitContext
WireAuth();
var cut = Render<SmsConfigurationPage>();
cut.WaitForState(() => cut.Markup.Contains("ACtest_account_sid"));
cut.WaitForState(() => cut.Markup.Contains("ACcccccccccccccccccccccccccccccccc"));
cut.FindAll("button").First(b => b.TextContent.Contains("Edit")).Click();
// Leave the (blank) Auth Token input untouched, then save.
@@ -193,7 +193,7 @@ public class SmsConfigurationPageTests : BunitContext
WireAuth();
var cut = Render<SmsConfigurationPage>();
cut.WaitForState(() => cut.Markup.Contains("ACtest_account_sid"));
cut.WaitForState(() => cut.Markup.Contains("ACcccccccccccccccccccccccccccccccc"));
cut.FindAll("button").First(b => b.TextContent.Contains("Edit")).Click();
cut.FindAll("input[type=password]").First().Change("rotated-token");
@@ -0,0 +1,19 @@
using ZB.MOM.WW.ScadaBridge.Commons.Entities.ExternalSystems;
namespace ZB.MOM.WW.ScadaBridge.Commons.Tests.Entities;
/// <summary>
/// Locks the <see cref="ExternalSystemDefinition"/> construction invariants — in particular
/// that the per-system <see cref="ExternalSystemDefinition.TimeoutSeconds"/> defaults to 0,
/// meaning "unset — use the gateway's configured DefaultHttpTimeout".
/// </summary>
public class ExternalSystemDefinitionTests
{
[Fact]
public void ExternalSystemDefinition_TimeoutSeconds_DefaultsToZeroMeaningUseGatewayDefault()
{
var system = new ExternalSystemDefinition("sys", "http://x", "none");
Assert.Equal(0, system.TimeoutSeconds);
}
}
@@ -45,6 +45,21 @@ public class ExternalSystemRepositoryTests : IDisposable
Assert.Equal("Sys", loaded!.Name);
}
[Fact]
public async Task AddExternalSystem_TimeoutSeconds_RoundTrips()
{
var def = new ExternalSystemDefinition("Sys", "https://example.test", "ApiKey")
{
TimeoutSeconds = 15,
};
await _repository.AddExternalSystemAsync(def);
await _repository.SaveChangesAsync();
var loaded = await _repository.GetExternalSystemByIdAsync(def.Id);
Assert.NotNull(loaded);
Assert.Equal(15, loaded!.TimeoutSeconds);
}
[Fact]
public async Task GetMethodsByExternalSystemId_FiltersByParent()
{
@@ -64,4 +64,13 @@ public class ErrorClassifierTests
Assert.IsType<TransientExternalSystemException>(ex);
Assert.Equal("test message", ex.Message);
}
[Fact]
public void IsTransient_CallerCancelledOce_IsNotTransient()
{
var cts = new CancellationTokenSource();
cts.Cancel();
Assert.False(ErrorClassifier.IsTransient(new OperationCanceledException(), cts.Token));
Assert.True(ErrorClassifier.IsTransient(new OperationCanceledException(), CancellationToken.None));
}
}
@@ -828,6 +828,105 @@ public class ExternalSystemClientTests
Assert.Equal("alice:s3cret", decoded);
}
[Fact]
public async Task ApiKeyAuth_JsonConfig_SetsConfiguredHeaderAndKeepsColons()
{
var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "apikey")
{
Id = 1,
AuthConfiguration = "{\"header\":\"X-Auth\",\"key\":\"ab:cd:ef\"}",
};
var method = new ExternalSystemMethod("getData", "GET", "/data") { Id = 1, ExternalSystemDefinitionId = 1 };
StubResolution(system, method);
var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
_httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
await client.CallAsync("TestAPI", "getData");
Assert.True(handler.LastHeaders!.TryGetValues("X-Auth", out var values));
Assert.Equal("ab:cd:ef", values!.Single());
}
[Fact]
public async Task ApiKeyAuth_JsonConfig_DefaultHeader()
{
// The UI-placeholder shape ({"key":"xyz"}) — silently misparsed by the
// legacy colon-split before Task 15 (it produced header name '{"key').
var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "apikey")
{
Id = 1,
AuthConfiguration = "{\"key\":\"xyz\"}",
};
var method = new ExternalSystemMethod("getData", "GET", "/data") { Id = 1, ExternalSystemDefinitionId = 1 };
StubResolution(system, method);
var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
_httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
await client.CallAsync("TestAPI", "getData");
Assert.True(handler.LastHeaders!.TryGetValues("X-API-Key", out var values));
Assert.Equal("xyz", values!.Single());
}
[Fact]
public async Task BasicAuth_JsonConfig_EncodesUsernamePassword()
{
var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "basic")
{
Id = 1,
AuthConfiguration = "{\"username\":\"u\",\"password\":\"p:w\"}",
};
var method = new ExternalSystemMethod("getData", "GET", "/data") { Id = 1, ExternalSystemDefinitionId = 1 };
StubResolution(system, method);
var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
_httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
await client.CallAsync("TestAPI", "getData");
var auth = handler.LastHeaders!.Authorization;
Assert.NotNull(auth);
Assert.Equal("Basic", auth!.Scheme);
var decoded = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(auth.Parameter!));
Assert.Equal("u:p:w", decoded);
}
[Fact]
public async Task ApiKeyAuth_LegacyColonConfig_StillWorks()
{
// Backward compatibility: the legacy "HeaderName:KeyValue" colon form must
// keep working for existing rows.
var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "apikey")
{
Id = 1,
AuthConfiguration = "X-Custom:secret",
};
var method = new ExternalSystemMethod("getData", "GET", "/data") { Id = 1, ExternalSystemDefinitionId = 1 };
StubResolution(system, method);
var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
_httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
await client.CallAsync("TestAPI", "getData");
Assert.True(handler.LastHeaders!.TryGetValues("X-Custom", out var values));
Assert.Equal("secret", values!.Single());
}
[Fact]
public async Task Call_ConnectionError_IsClassifiedAsTransient()
{
@@ -872,6 +971,33 @@ public class ExternalSystemClientTests
e.Level >= LogLevel.Warning && e.Message.Contains("TestAPI"));
}
[Fact]
public async Task PerSystemTimeout_OverridesGatewayDefault()
{
// System timeout 1s; handler black-holes far longer; gateway default is 30s.
var system = new ExternalSystemDefinition("slow", "http://x", "none") { Id = 1, TimeoutSeconds = 1 };
var method = new ExternalSystemMethod("m", "GET", "/p") { Id = 1, ExternalSystemDefinitionId = 1 };
StubResolution(system, method);
var httpClient = new HttpClient(new HangingHttpMessageHandler(TimeSpan.FromMinutes(10)));
_httpClientFactory.CreateClient(Arg.Any<string>()).Returns(httpClient);
// No options override → DefaultHttpTimeout is the 30s default; only the
// per-system 1s should govern the round-trip.
var client = new ExternalSystemClient(
_httpClientFactory, _repository,
NullLogger<ExternalSystemClient>.Instance);
var sw = System.Diagnostics.Stopwatch.StartNew();
var result = await client.CallAsync("slow", "m");
sw.Stop();
Assert.False(result.Success);
Assert.Contains("Timeout", result.ErrorMessage);
Assert.True(sw.Elapsed < TimeSpan.FromSeconds(4),
"per-system 1s timeout must apply, not the 30s default");
}
[Fact]
public async Task Call_TransientFailure_DoesNotLogAtWarningOrAbove()
{
@@ -915,6 +1041,59 @@ public class ExternalSystemClientTests
}
}
// ── ESG: bound outbound response buffering (MaxResponseBodyBytes) ──
[Fact]
public async Task OversizedResponseBody_FailsPermanently_WithoutBufferingWholeBody()
{
var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
var method = new ExternalSystemMethod("postData", "POST", "/post") { Id = 1, ExternalSystemDefinitionId = 1 };
StubResolution(system, method);
// The endpoint returns a 2 MiB body — far above the 1 KiB cap below.
// Hand out a fresh HttpClient per CreateClient call: the client mutates
// HttpClient.Timeout before its first request, so a single shared instance
// cannot serve the two calls this test makes.
var hugeBody = new string('x', 2 * 1024 * 1024);
_httpClientFactory.CreateClient(Arg.Any<string>())
.Returns(_ => new HttpClient(new MockHttpMessageHandler(HttpStatusCode.OK, hugeBody)));
// A real S&F service so we can assert the oversized response is NOT buffered.
var dbName = $"EsgOversize_{Guid.NewGuid():N}";
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
using var keepAlive = new SqliteConnection(connStr);
keepAlive.Open();
var storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
await storage.InitializeAsync();
var sfOptions = new StoreAndForwardOptions
{
DefaultRetryInterval = TimeSpan.FromMinutes(10),
RetryTimerInterval = TimeSpan.FromMinutes(10),
};
var sf = new StoreAndForwardService(storage, sfOptions, NullLogger<StoreAndForwardService>.Instance);
var options = new ExternalSystemGatewayOptions { MaxResponseBodyBytes = 1024 };
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance,
storeAndForward: sf,
options: Microsoft.Extensions.Options.Options.Create(options));
// Sync call: oversize is a permanent failure — retrying will not shrink it.
var result = await client.CallAsync("TestAPI", "postData");
Assert.False(result.Success);
Assert.Contains("exceeded", result.ErrorMessage!, StringComparison.OrdinalIgnoreCase);
// Cached call of the same shape: a permanent failure must NOT be buffered
// into S&F (buffering it would defeat the response-size cap).
var cached = await client.CachedCallAsync("TestAPI", "postData");
Assert.False(cached.WasBuffered);
var depth = await storage.GetBufferDepthByCategoryAsync();
Assert.False(
depth.TryGetValue(ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.StoreAndForwardCategory.ExternalSystem, out var n) && n > 0,
"An oversized (permanent-failure) response must not be buffered for retry");
}
/// <summary>
/// Test helper: mock HTTP message handler.
/// </summary>
@@ -1087,4 +1266,79 @@ public class ExternalSystemClientTests
Assert.True(result.Success);
}
// ── ExternalSystemGateway: {param} path-template substitution ──
[Fact]
public async Task PathTemplate_SubstitutesAndConsumesParameter()
{
// A `{id}` placeholder in the method path is replaced with the escaped
// parameter value and the consumed name is removed from the query string.
var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
var method = new ExternalSystemMethod("getRecipe", "GET", "/recipes/{id}") { Id = 1, ExternalSystemDefinitionId = 1 };
StubResolution(system, method);
var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
_httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
await client.CallAsync("TestAPI", "getRecipe", new Dictionary<string, object?>
{
["id"] = "R 1",
["filter"] = "x",
});
var uri = handler.LastUri!.AbsoluteUri;
// The value is substituted into the path (escaped) and consumed — it must
// NOT reappear in the query string; the remaining parameter still does.
Assert.Equal("https://api.example.com/recipes/R%201?filter=x", uri);
Assert.DoesNotContain("id=", uri);
}
[Fact]
public async Task PathTemplate_Post_ConsumedParamExcludedFromBody()
{
// A path-template parameter consumed by a POST path must not also be
// serialized into the JSON body.
var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
var method = new ExternalSystemMethod("addLine", "POST", "/orders/{orderId}/lines") { Id = 1, ExternalSystemDefinitionId = 1 };
StubResolution(system, method);
var handler = new RequestCapturingHandler(HttpStatusCode.OK, "{}");
_httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(handler));
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
await client.CallAsync("TestAPI", "addLine", new Dictionary<string, object?>
{
["orderId"] = 5,
["qty"] = 2,
});
Assert.Equal("https://api.example.com/orders/5/lines", handler.LastUri!.ToString());
Assert.Contains("\"qty\":2", handler.LastBody);
Assert.DoesNotContain("orderId", handler.LastBody);
}
[Fact]
public async Task PathTemplate_MissingParameter_ThrowsClearError()
{
// A `{id}` placeholder with no matching parameter is an authoring error —
// reject it with a clear ArgumentException naming the placeholder.
var system = new ExternalSystemDefinition("TestAPI", "https://api.example.com", "none") { Id = 1 };
var method = new ExternalSystemMethod("getRecipe", "GET", "/recipes/{id}") { Id = 1, ExternalSystemDefinitionId = 1 };
StubResolution(system, method);
_httpClientFactory.CreateClient(Arg.Any<string>()).Returns(new HttpClient(new RequestCapturingHandler(HttpStatusCode.OK, "{}")));
var client = new ExternalSystemClient(
_httpClientFactory, _repository, NullLogger<ExternalSystemClient>.Instance);
var ex = await Assert.ThrowsAsync<ArgumentException>(
() => client.CallAsync("TestAPI", "getRecipe"));
Assert.Contains("id", ex.Message);
}
}
@@ -92,6 +92,59 @@ public sealed class EndpointContentTypeTests : IDisposable
Assert.Contains("42", body);
}
[Fact]
public async Task NonJsonContentType_WithBody_Returns415()
{
// Task 5 (S9): a body with a non-JSON Content-Type must be rejected with 415 +
// UNSUPPORTED_MEDIA_TYPE rather than a misleading 400 "invalid JSON". A body
// with NO Content-Type header is still parsed leniently as JSON (preserving
// existing callers).
const string methodName = "echo415";
var method = new ApiMethod(methodName, "return Parameters[\"value\"];")
{
Id = 1,
TimeoutSeconds = 10,
ParameterDefinitions = """[{"name":"value","type":"Integer","required":true}]""",
};
var repo = Substitute.For<IInboundApiRepository>();
repo.GetMethodByNameAsync(methodName, Arg.Any<CancellationToken>())
.Returns(method);
using var host = await BuildHostAsync(repo);
var token = await SeedKeyAsync(host, methodName);
var client = host.GetTestClient();
// (a) text/plain body → 415 UNSUPPORTED_MEDIA_TYPE
var nonJson = new HttpRequestMessage(HttpMethod.Post, "/api/" + methodName)
{
Content = new ByteArrayContent(Encoding.UTF8.GetBytes("hello")),
};
nonJson.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("text/plain");
nonJson.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
var nonJsonResponse = await client.SendAsync(nonJson);
var nonJsonBody = await nonJsonResponse.Content.ReadAsStringAsync();
Assert.Equal(HttpStatusCode.UnsupportedMediaType, nonJsonResponse.StatusCode);
Assert.Contains("UNSUPPORTED_MEDIA_TYPE", nonJsonBody);
// (b) body with NO Content-Type header → still parses as JSON (lenient), 200.
var noContentType = new HttpRequestMessage(HttpMethod.Post, "/api/" + methodName)
{
Content = new ByteArrayContent(Encoding.UTF8.GetBytes("{\"value\":42}")),
};
noContentType.Content.Headers.ContentType = null;
noContentType.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
var lenientResponse = await client.SendAsync(noContentType);
var lenientBody = await lenientResponse.Content.ReadAsStringAsync();
Assert.Equal(HttpStatusCode.OK, lenientResponse.StatusCode);
Assert.Contains("42", lenientBody);
}
/// <summary>Seeds a key scoped for <paramref name="methodName"/> and returns its Bearer token.</summary>
private static async Task<string> SeedKeyAsync(IHost host, string methodName)
{
@@ -26,6 +26,7 @@ using System.Diagnostics.Metrics;
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
namespace ZB.MOM.WW.ScadaBridge.InboundAPI.Tests;
@@ -67,8 +68,11 @@ public sealed class EndpointExtensionsTests : IDisposable
// two bodies are equal-to-each-other) catches a single-branch divergence where
// BOTH branches change in lockstep but away from the agreed enumeration-safe
// body — the equal-to-each-other check alone would still pass in that case.
// Pinned to EndpointExtensions.NotApprovedMessage's JSON shape.
private const string NotApprovedBodyJson = """{"error":"API key not approved for this method"}""";
// Pinned to EndpointExtensions.NotApprovedMessage's JSON shape. Task 5 (C3): the
// error body now additionally carries the machine-readable "code" field; both 403
// branches still emit this identical body, preserving enumeration-safety.
private const string NotApprovedBodyJson =
"""{"error":"API key not approved for this method","code":"NOT_APPROVED"}""";
// Each test gets its own throwaway SQLite database so seeded keys never leak
// between tests; the file is deleted on Dispose.
@@ -341,6 +345,45 @@ public sealed class EndpointExtensionsTests : IDisposable
Assert.Equal(NotApprovedBodyJson, notInScopeBody);
}
[Fact]
public async Task ErrorBodies_CarryMachineReadableCode()
{
// Task 5 (C3): every error body carries a machine-readable "code" field for
// external integrators — UNAUTHORIZED (401), INVALID_JSON (400),
// VALIDATION_FAILED (400).
var method = SeedMethod(1, "needsParam", "return Parameters[\"value\"];",
"""[{"name":"value","type":"Integer","required":true}]""");
using var host = await BuildHostAsync(method);
var token = await SeedKeyAsync(host, "key1", "caller", new[] { "needsParam" });
var client = host.GetTestClient();
// (a) missing credential → UNAUTHORIZED
var unauth = await client.SendAsync(new HttpRequestMessage(HttpMethod.Post, "/api/needsParam")
{
Content = new StringContent("{}", Encoding.UTF8, "application/json"),
});
Assert.Equal(HttpStatusCode.Unauthorized, unauth.StatusCode);
Assert.Equal("UNAUTHORIZED", await ReadCodeAsync(unauth));
// (b) in-scope but invalid JSON → INVALID_JSON
var badJson = await client.SendAsync(BuildPost("needsParam", "{ not json", token));
Assert.Equal(HttpStatusCode.BadRequest, badJson.StatusCode);
Assert.Equal("INVALID_JSON", await ReadCodeAsync(badJson));
// (c) missing required parameter → VALIDATION_FAILED
var missing = await client.SendAsync(BuildPost("needsParam", "{}", token));
Assert.Equal(HttpStatusCode.BadRequest, missing.StatusCode);
Assert.Equal("VALIDATION_FAILED", await ReadCodeAsync(missing));
}
private static async Task<string?> ReadCodeAsync(HttpResponseMessage response)
{
var body = await response.Content.ReadAsStringAsync();
using var doc = JsonDocument.Parse(body);
return doc.RootElement.TryGetProperty("code", out var code) ? code.GetString() : null;
}
[Fact]
public async Task InvalidJsonBody_Returns400()
{
@@ -15,7 +15,7 @@ public class EndpointGatingTests
{
private static InboundApiEndpointFilter CreateFilter(InboundApiOptions? options = null) =>
new(NullLogger<InboundApiEndpointFilter>.Instance,
Options.Create(options ?? new InboundApiOptions()));
new TestOptionsMonitor<InboundApiOptions>(options ?? new InboundApiOptions()));
private static (DefaultHttpContext ctx, DefaultEndpointFilterInvocationContext invocation)
BuildInvocation(bool? activeNode, long? contentLength)
@@ -144,4 +144,37 @@ public class EndpointGatingTests
public bool IsReadOnly => false;
public long? MaxRequestBodySize { get; set; }
}
// --- InboundAPI hot-reload: options must be re-read per request, not cached at ctor time ---
[Fact]
public async Task BodySizeCap_HotReloads_WithoutRestart()
{
var monitor = new TestOptionsMonitor<InboundApiOptions>(new InboundApiOptions { MaxRequestBodyBytes = 10 });
var filter = new InboundApiEndpointFilter(NullLogger<InboundApiEndpointFilter>.Instance, monitor);
var (_, firstInvocation) = BuildInvocation(activeNode: true, contentLength: 20);
var firstNext = NextSentinel(out var firstHandlerRan);
var firstResult = await filter.InvokeAsync(firstInvocation, firstNext);
Assert.False(firstHandlerRan());
var firstStatus = Assert.IsAssignableFrom<IStatusCodeHttpResult>(firstResult);
Assert.Equal(StatusCodes.Status413PayloadTooLarge, firstStatus.StatusCode);
monitor.CurrentValue = new InboundApiOptions { MaxRequestBodyBytes = 1024 * 1024 };
var (_, secondInvocation) = BuildInvocation(activeNode: true, contentLength: 20);
var secondNext = NextSentinel(out var secondHandlerRan);
await filter.InvokeAsync(secondInvocation, secondNext);
Assert.True(secondHandlerRan());
}
private sealed class TestOptionsMonitor<T> : IOptionsMonitor<T>
{
public TestOptionsMonitor(T currentValue) => CurrentValue = currentValue;
public T CurrentValue { get; set; }
public T Get(string? name) => CurrentValue;
public IDisposable? OnChange(Action<T, string?> listener) => null;
}
}
@@ -0,0 +1,106 @@
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.InboundApi;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
namespace ZB.MOM.WW.ScadaBridge.InboundAPI.Tests;
/// <summary>
/// Arch-review S1: the compiled-handler cache must be revision-checked against the
/// freshly-fetched ApiMethod.Script so a standby node (whose cache was populated at
/// its own startup) never serves a stale delegate after failover, and a method fixed
/// out-of-band is never stuck behind a stale known-bad record.
/// </summary>
public class InboundScriptExecutorStalenessTests
{
private readonly InboundScriptExecutor _executor;
private readonly RouteHelper _route;
public InboundScriptExecutorStalenessTests()
{
_executor = new InboundScriptExecutor(
NullLogger<InboundScriptExecutor>.Instance, Substitute.For<IServiceProvider>());
_route = new RouteHelper(
Substitute.For<IInstanceLocator>(), Substitute.For<IInstanceRouter>());
}
private Task<InboundScriptResult> Run(ApiMethod m) => _executor.ExecuteAsync(
m, new Dictionary<string, object?>(), _route, TimeSpan.FromSeconds(10));
[Fact]
public async Task UpdatedScript_RecompilesInsteadOfServingStaleHandler()
{
// Simulates the failover scenario: this node compiled v1 at startup...
var v1 = new ApiMethod("stale", "return 1;") { Id = 1, TimeoutSeconds = 10 };
Assert.True(_executor.CompileAndRegister(v1));
var r1 = await Run(v1);
Assert.Contains("1", r1.ResultJson);
// ...the active node updated the method (this node never heard about it);
// the per-request DB fetch hands ExecuteAsync the NEW row.
var v2 = new ApiMethod("stale", "return 2;") { Id = 1, TimeoutSeconds = 10 };
var r2 = await Run(v2);
Assert.True(r2.Success);
Assert.Contains("2", r2.ResultJson); // old cache would have returned 1
}
[Fact]
public async Task KnownBadMethod_FixedScript_CompilesInsteadOfFastFailing()
{
// The mirror-image variant: broken at this node's startup...
var broken = new ApiMethod("fixme", "%%% not C# %%%") { Id = 2, TimeoutSeconds = 10 };
Assert.False(_executor.CompileAndRegister(broken));
var r1 = await Run(broken);
Assert.False(r1.Success);
// ...fixed via the management path on the OTHER node; this node's fresh
// DB fetch carries the fixed script.
var fixedMethod = new ApiMethod("fixme", "return 42;") { Id = 2, TimeoutSeconds = 10 };
var r2 = await Run(fixedMethod);
Assert.True(r2.Success);
Assert.Contains("42", r2.ResultJson);
}
[Fact]
public async Task KnownBadMethod_SameScript_StillFastFails()
{
var broken = new ApiMethod("stillbad", "%%% not C# %%%") { Id = 3, TimeoutSeconds = 10 };
Assert.False(_executor.CompileAndRegister(broken));
// Same script text → the fast-fail record must hold (no per-request Roslyn).
var again = new ApiMethod("stillbad", "%%% not C# %%%") { Id = 3, TimeoutSeconds = 10 };
var r = await Run(again);
Assert.False(r.Success);
Assert.Contains("Script compilation failed", r.ErrorMessage);
}
[Fact]
public async Task TestSeamRegisterHandler_WithoutScript_IsNotRevisionChecked()
{
// RegisterHandler (script-less test seam) pins the handler regardless of row content.
var m = new ApiMethod("pinned", "return 0;") { Id = 4, TimeoutSeconds = 10 };
_executor.RegisterHandler("pinned", async _ => { await Task.CompletedTask; return 99; });
var r = await Run(m);
Assert.True(r.Success);
Assert.Contains("99", r.ResultJson);
}
[Fact]
public async Task InvalidateMethod_DropsHandlerAndKnownBadRecord()
{
var good = new ApiMethod("inv", "return 5;") { Id = 5, TimeoutSeconds = 10 };
Assert.True(_executor.CompileAndRegister(good));
var broken = new ApiMethod("invbad", "%%%") { Id = 6, TimeoutSeconds = 10 };
Assert.False(_executor.CompileAndRegister(broken));
_executor.InvalidateMethod("inv");
_executor.InvalidateMethod("invbad");
Assert.Equal(0, _executor.KnownBadMethodCount);
// After invalidation the method lazily recompiles from the row — still serves.
var r = await Run(good);
Assert.True(r.Success);
}
}
@@ -97,6 +97,32 @@ public class InboundScriptExecutorTests
Assert.Equal("Internal script error", result.ErrorMessage);
}
[Fact]
public async Task RoutedSiteUnreachable_Returns500WithSiteUnreachableCode()
{
// C3: a routed call whose site is unreachable must surface the machine-readable
// SITE_UNREACHABLE code (spec Component-InboundAPI failure body) rather than
// collapsing into a generic SCRIPT_ERROR.
var locator = Substitute.For<IInstanceLocator>();
locator.GetSiteIdForInstanceAsync("X", Arg.Any<CancellationToken>()).Returns("SiteA");
var router = Substitute.For<IInstanceRouter>();
router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToCallResponse(
((RouteToCallRequest)ci[1]).CorrelationId,
Success: false, ReturnValue: null, ErrorMessage: "Site unreachable",
DateTimeOffset.UtcNow));
var route = new RouteHelper(locator, router);
var method = new ApiMethod("routed", "return await Route.To(\"X\").Call(\"s\");")
{ Id = 7, TimeoutSeconds = 10 };
var result = await _executor.ExecuteAsync(
method, new Dictionary<string, object?>(), route, TimeSpan.FromSeconds(10));
Assert.False(result.Success);
Assert.Equal("SITE_UNREACHABLE", result.ErrorCode);
}
[Fact]
public async Task HandlerTimesOut_ReturnsTimeoutError()
{
@@ -271,12 +297,15 @@ public class InboundScriptExecutorTests
}
[Fact]
public async Task CompileAndRegister_NonCompilingUpdate_KeepsPreviousHandler()
public async Task CompileAndRegister_NonCompilingUpdate_ExecuteSurfacesCompileFailure()
{
// Register a working handler, then "update" the same method name with a script
// that does not compile. The broken save must NOT replace the working delegate —
// the previously registered version keeps serving requests (this is exactly the
// behaviour that makes a broken save non-fatal while the warning is surfaced).
// Arch-review S1 (DB-authoritative): CompileAndRegister still leaves the old
// handler cached on a broken save (it never overwrites with a bad compile), but
// ExecuteAsync now revision-checks the cached script against the freshly-fetched
// row. A request carrying the broken row no longer silently serves the stale good
// delegate — the mismatch triggers a recompile, which fails, so the method returns
// a compilation error consistently on every node (honest failure over node-divergent
// stale success).
var good = new ApiMethod("evolving", "return 111;") { Id = 1, TimeoutSeconds = 10 };
Assert.True(_executor.CompileAndRegister(good, out var goodErrors));
Assert.Empty(goodErrors);
@@ -285,12 +314,19 @@ public class InboundScriptExecutorTests
Assert.False(_executor.CompileAndRegister(broken, out var brokenErrors));
Assert.NotEmpty(brokenErrors);
// The old (good) handler still serves — the broken script never went live.
// The DB row is authoritative: the broken script now 500s rather than the old
// delegate silently serving.
var result = await _executor.ExecuteAsync(
broken, new Dictionary<string, object?>(), _route, TimeSpan.FromSeconds(10));
Assert.True(result.Success);
Assert.Contains("111", result.ResultJson);
Assert.False(result.Success);
Assert.Contains("Script compilation failed", result.ErrorMessage);
// The still-valid row keeps serving — only the broken revision fails.
var stillGood = await _executor.ExecuteAsync(
good, new Dictionary<string, object?>(), _route, TimeSpan.FromSeconds(10));
Assert.True(stillGood.Success);
Assert.Contains("111", stillGood.ResultJson);
}
// --- InboundAPI-002: lazy compile-and-fetch must be atomic, never KeyNotFoundException ---
@@ -718,6 +754,26 @@ public class InboundScriptExecutorTests
return new SqliteGateway("DataSource=file:movein-exec?mode=memory&cache=shared");
}
[Fact]
public async Task TimedOutScript_ScopeDisposalDeferred_NoOrphanLeak()
{
// Handler ignores the token and keeps running past the method timeout.
var release = new TaskCompletionSource();
_executor.RegisterHandler("slow", async _ => { await release.Task; return 1; });
var method = new ApiMethod("slow", "unused") { Id = 9, TimeoutSeconds = 1 };
var result = await _executor.ExecuteAsync(
method, new Dictionary<string, object?>(), _route, TimeSpan.FromMilliseconds(50));
Assert.False(result.Success);
Assert.Contains("timed out", result.ErrorMessage);
Assert.Equal(1, _executor.AbandonedExecutionCount); // orphan is COUNTED
release.SetResult(); // orphan finishes; continuation cleans up
await Task.Delay(200);
Assert.Equal(0, _executor.AbandonedExecutionCount); // ...and DECREMENTED
}
private sealed class CompileLogCounter
{
public int CompilationFailures;
@@ -417,8 +417,9 @@ public class ManagementActorTests : TestKit, IDisposable
{
// The save still persists, but a script that does not compile must surface a
// top-level compileWarning in the result rather than silently looking like a
// success — and the previously registered handler keeps serving (CompileAndRegister
// never replaces a working delegate with a broken one).
// success. The DB row is authoritative: ExecuteAsync revision-checks the cached
// handler against the fresh row, so the broken script returns a compilation error
// on every request/node rather than a previously registered handler still serving.
var existing = new Commons.Entities.InboundApi.ApiMethod("MyMethod", "return 1;")
{
Id = 7,
@@ -1403,6 +1404,65 @@ public class ManagementActorTests : TestKit, IDisposable
Assert.Equal("Basic", existing.AuthType);
}
[Fact]
public void UpdateSmtpConfig_AppliesOAuth2AuthorityAndScope()
{
var notifRepo = Substitute.For<INotificationRepository>();
var existing = new Commons.Entities.Notifications.SmtpConfiguration(
"old.example.com", "OAuth2", "old@example.com")
{
Id = 1,
Port = 25,
OAuth2Authority = "https://old.authority/token",
OAuth2Scope = "old.scope/.default",
};
notifRepo.GetSmtpConfigurationByIdAsync(1, Arg.Any<CancellationToken>()).Returns(existing);
_services.AddScoped(_ => notifRepo);
var actor = CreateActor();
var envelope = Envelope(
new UpdateSmtpConfigCommand(
1, "new.example.com", 465, "OAuth2", "new@example.com",
OAuth2Authority: "https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token",
OAuth2Scope: "https://outlook.office365.com/.default"),
"Administrator");
actor.Tell(envelope);
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
Assert.Equal(envelope.CorrelationId, response.CorrelationId);
Assert.Equal("https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token", existing.OAuth2Authority);
Assert.Equal("https://outlook.office365.com/.default", existing.OAuth2Scope);
}
[Fact]
public void UpdateSmtpConfig_WithNullOAuth2AuthorityAndScope_PreservesExistingValues()
{
var notifRepo = Substitute.For<INotificationRepository>();
var existing = new Commons.Entities.Notifications.SmtpConfiguration(
"old.example.com", "OAuth2", "old@example.com")
{
Id = 1,
Port = 25,
OAuth2Authority = "https://old.authority/token",
OAuth2Scope = "old.scope/.default",
};
notifRepo.GetSmtpConfigurationByIdAsync(1, Arg.Any<CancellationToken>()).Returns(existing);
_services.AddScoped(_ => notifRepo);
var actor = CreateActor();
var envelope = Envelope(
new UpdateSmtpConfigCommand(1, "new.example.com", 465, "OAuth2", "new@example.com"),
"Administrator");
actor.Tell(envelope);
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
Assert.Equal(envelope.CorrelationId, response.CorrelationId);
Assert.Equal("https://old.authority/token", existing.OAuth2Authority);
Assert.Equal("old.scope/.default", existing.OAuth2Scope);
}
// ========================================================================
// SMS Notifications (S5) — list Type discriminator + SMS-config management
// ========================================================================
@@ -1598,7 +1658,7 @@ public class ManagementActorTests : TestKit, IDisposable
var actor = CreateActor();
var envelope = Envelope(
new UpdateSmsConfigCommand(1, "ACnew", "+15551110000", "MGnew",
new UpdateSmsConfigCommand(1, "ACbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "+15551110000", "MGnew",
"https://new.example.com", "new-secret"),
"Administrator");
@@ -1606,7 +1666,7 @@ public class ManagementActorTests : TestKit, IDisposable
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
Assert.Equal(envelope.CorrelationId, response.CorrelationId);
Assert.Equal("ACnew", existing.AccountSid);
Assert.Equal("ACbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", existing.AccountSid);
Assert.Equal("+15551110000", existing.FromNumber);
Assert.Equal("MGnew", existing.MessagingServiceSid);
Assert.Equal("https://new.example.com", existing.ApiBaseUrl);
@@ -1630,7 +1690,7 @@ public class ManagementActorTests : TestKit, IDisposable
var actor = CreateActor();
// AuthToken + ApiBaseUrl omitted -> preserve-if-null.
var envelope = Envelope(
new UpdateSmsConfigCommand(1, "ACnew", "+15551110000"),
new UpdateSmsConfigCommand(1, "ACbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "+15551110000"),
"Administrator");
actor.Tell(envelope);
@@ -1641,7 +1701,7 @@ public class ManagementActorTests : TestKit, IDisposable
Assert.Equal("old-secret", existing.AuthToken);
Assert.Equal("https://old.example.com", existing.ApiBaseUrl);
// Provided fields are still updated.
Assert.Equal("ACnew", existing.AccountSid);
Assert.Equal("ACbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", existing.AccountSid);
Assert.Equal("+15551110000", existing.FromNumber);
}
@@ -1666,7 +1726,7 @@ public class ManagementActorTests : TestKit, IDisposable
var actor = CreateActor();
var envelope = Envelope(
new UpdateSmsConfigCommand(1, "ACnew", "+15551110000", AuthToken: "super-secret-token"),
new UpdateSmsConfigCommand(1, "ACbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "+15551110000", AuthToken: "super-secret-token"),
"Administrator");
actor.Tell(envelope);
@@ -1714,7 +1774,7 @@ public class ManagementActorTests : TestKit, IDisposable
// cannot reach it.
var actor = CreateActor();
var envelope = Envelope(
new UpdateSmsConfigCommand(1, "ACnew", "+15551110000"),
new UpdateSmsConfigCommand(1, "ACbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "+15551110000"),
"Viewer");
actor.Tell(envelope);
@@ -1731,7 +1791,7 @@ public class ManagementActorTests : TestKit, IDisposable
// could rotate a production Twilio Auth Token via the CLI/Management API.
var actor = CreateActor();
var envelope = Envelope(
new UpdateSmsConfigCommand(1, "ACnew", "+15551110000", AuthToken: "rotate-me"),
new UpdateSmsConfigCommand(1, "ACbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "+15551110000", AuthToken: "rotate-me"),
"Designer");
actor.Tell(envelope);
@@ -1758,7 +1818,7 @@ public class ManagementActorTests : TestKit, IDisposable
var actor = CreateActor();
var envelope = Envelope(
new UpdateSmsConfigCommand(1, "ACnew", "+15551110000", AuthToken: " "),
new UpdateSmsConfigCommand(1, "ACbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "+15551110000", AuthToken: " "),
"Administrator");
actor.Tell(envelope);
@@ -1768,7 +1828,7 @@ public class ManagementActorTests : TestKit, IDisposable
// The blank token was ignored; the stored secret survives.
Assert.Equal("old-secret", existing.AuthToken);
// Non-secret fields still updated.
Assert.Equal("ACnew", existing.AccountSid);
Assert.Equal("ACbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", existing.AccountSid);
}
[Fact]
@@ -1788,7 +1848,7 @@ public class ManagementActorTests : TestKit, IDisposable
var actor = CreateActor();
var envelope = Envelope(
new UpdateSmsConfigCommand(1, "ACnew", FromNumber: null, MessagingServiceSid: "MGonly"),
new UpdateSmsConfigCommand(1, "ACbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", FromNumber: null, MessagingServiceSid: "MGonly"),
"Administrator");
actor.Tell(envelope);
@@ -1799,6 +1859,63 @@ public class ManagementActorTests : TestKit, IDisposable
Assert.Equal("MGonly", existing.MessagingServiceSid);
}
[Fact]
public void UpdateSmsConfig_MalformedAccountSid_Rejected()
{
// Task 22: AccountSid is interpolated un-escaped into the Twilio request URI,
// so a malformed SID (e.g. a path-traversal / query-injection payload) must be
// rejected at save time, closing the URI-injection door before it can be persisted.
var notifRepo = Substitute.For<INotificationRepository>();
var existing = new Commons.Entities.Notifications.SmsConfiguration("ACaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "+15550000000")
{
Id = 1,
AuthToken = "old-secret",
};
notifRepo.GetAllSmsConfigurationsAsync(Arg.Any<CancellationToken>())
.Returns(new List<Commons.Entities.Notifications.SmsConfiguration> { existing });
_services.AddScoped(_ => notifRepo);
var actor = CreateActor();
var envelope = Envelope(
new UpdateSmsConfigCommand(1, "../evil?x=", "+15551110000"),
"Administrator");
actor.Tell(envelope);
var response = ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5));
Assert.Equal("COMMAND_FAILED", response.ErrorCode);
Assert.Contains("Account SID", response.Error);
// The malformed value must never have been persisted.
Assert.Equal("ACaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", existing.AccountSid);
}
[Fact]
public void UpdateSmsConfig_ValidSid_Accepted()
{
// Task 22: a well-formed Twilio SID ('AC' + 32 hex chars) passes the guard.
var notifRepo = Substitute.For<INotificationRepository>();
var existing = new Commons.Entities.Notifications.SmsConfiguration("ACold", "+15550000000")
{
Id = 1,
AuthToken = "old-secret",
};
notifRepo.GetAllSmsConfigurationsAsync(Arg.Any<CancellationToken>())
.Returns(new List<Commons.Entities.Notifications.SmsConfiguration> { existing });
_services.AddScoped(_ => notifRepo);
var validSid = "AC" + new string('a', 32);
var actor = CreateActor();
var envelope = Envelope(
new UpdateSmsConfigCommand(1, validSid, "+15551110000"),
"Administrator");
actor.Tell(envelope);
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
Assert.Equal(envelope.CorrelationId, response.CorrelationId);
Assert.Equal(validSid, existing.AccountSid);
}
[Fact]
public void CuratedHandlerFailure_SurfacesTheCuratedMessage()
{
@@ -2922,4 +3039,56 @@ public class ManagementActorTests : TestKit, IDisposable
Assert.Contains("MissingSharedSchema", msg.JsonData, StringComparison.Ordinal);
Assert.Contains("\"category\":15", msg.JsonData, StringComparison.Ordinal);
}
// ========================================================================
// PLAN-06 Task 11: per-system ESG TimeoutSeconds management commands
// ========================================================================
[Fact]
public void CreateExternalSystem_PersistsTimeoutSeconds()
{
var extRepo = Substitute.For<IExternalSystemRepository>();
_services.AddScoped(_ => extRepo);
var actor = CreateActor();
var envelope = Envelope(
new CreateExternalSystemCommand("Sys", "http://x", "none", null, TimeoutSeconds: 15),
"Designer");
actor.Tell(envelope);
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
Assert.Equal(envelope.CorrelationId, response.CorrelationId);
extRepo.Received(1).AddExternalSystemAsync(
Arg.Is<Commons.Entities.ExternalSystems.ExternalSystemDefinition>(d => d.TimeoutSeconds == 15),
Arg.Any<CancellationToken>());
}
[Fact]
public void UpdateExternalSystem_NullTimeout_PreservesExisting()
{
var existing = new Commons.Entities.ExternalSystems.ExternalSystemDefinition("Sys", "http://x", "none")
{
Id = 7,
TimeoutSeconds = 42
};
var extRepo = Substitute.For<IExternalSystemRepository>();
extRepo.GetExternalSystemByIdAsync(7, Arg.Any<CancellationToken>()).Returns(existing);
_services.AddScoped(_ => extRepo);
var actor = CreateActor();
// TimeoutSeconds omitted (null) => existing 42 must be preserved.
var envelope = Envelope(
new UpdateExternalSystemCommand(7, "Sys", "http://x", "none", null),
"Designer");
actor.Tell(envelope);
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
Assert.Equal(envelope.CorrelationId, response.CorrelationId);
Assert.Equal(42, existing.TimeoutSeconds);
extRepo.Received(1).UpdateExternalSystemAsync(
Arg.Is<Commons.Entities.ExternalSystems.ExternalSystemDefinition>(d => d.TimeoutSeconds == 42),
Arg.Any<CancellationToken>());
}
}
@@ -264,4 +264,37 @@ public class EmailNotificationDeliveryAdapterTests
await Assert.ThrowsAnyAsync<OperationCanceledException>(
() => adapter.DeliverAsync(MakeNotification(), cts.Token));
}
[Fact]
public async Task MultipleSmtpConfigs_LowestIdWins()
{
// Arch-review S7: when more than one SMTP configuration row exists the pick
// must be deterministic (lowest Id), regardless of repository row order.
var list = new NotificationList("ops-team") { Id = 1 };
_repository.GetListByNameAsync("ops-team").Returns(list);
_repository.GetRecipientsByListIdAsync(1).Returns(new List<NotificationRecipient>
{
new("Alice", "alice@example.com") { Id = 1, NotificationListId = 1 }
});
// Repository returns the higher-Id row FIRST — FirstOrDefault would pick it.
_repository.GetAllSmtpConfigurationsAsync().Returns(new List<SmtpConfiguration>
{
new("smtp-seven.example.com", "basic", "noreply@example.com")
{
Id = 7, Port = 587, Credentials = "user:pass", TlsMode = "starttls"
},
new("smtp-three.example.com", "basic", "noreply@example.com")
{
Id = 3, Port = 587, Credentials = "user:pass", TlsMode = "starttls"
}
});
var adapter = CreateAdapter();
await adapter.DeliverAsync(MakeNotification());
// The lowest-Id row (3) must be the one connected to.
await _smtpClient.Received().ConnectAsync(
"smtp-three.example.com", Arg.Any<int>(), Arg.Any<SmtpTlsMode>(),
Arg.Any<int>(), Arg.Any<CancellationToken>());
}
}
@@ -299,6 +299,24 @@ public class SmsNotificationDeliveryAdapterTests
Assert.Equal(DeliveryResult.TransientFailure, outcome.Result);
}
[Fact]
public async Task MidListTransient_ShortCircuits_RemainingRecipientsNotAttempted()
{
// Three recipients; the FIRST one comes back transient (500). Every recipient
// accepted after a transient is a guaranteed duplicate on retry (the whole
// notification re-sends), so the adapter must stop at the first transient
// instead of attempting the remaining two.
SetupList(phones: new[] { "+15551112222", "+15553334444", "+15555556666" });
var handler = ScriptedHttpMessageHandler.ForStatuses(
HttpStatusCode.InternalServerError, HttpStatusCode.Created, HttpStatusCode.Created);
var adapter = CreateAdapter(handler);
var outcome = await adapter.DeliverAsync(MakeNotification());
Assert.Equal(DeliveryResult.TransientFailure, outcome.Result);
Assert.Single(handler.Requests);
}
[Fact]
public async Task Deliver_ListNotFound_ReturnsPermanent()
{
@@ -684,4 +684,78 @@ public class NotificationOutboxActorDispatchTests : TestKit
// Strictly sequential: never more than one delivery in flight at once.
Assert.Equal(1, adapter.HighWater);
}
// ── Task 18: per-type retry policy (SMS honors SmsConfiguration) ──────────
private void SetupSmsRetryPolicy(int maxRetries, TimeSpan retryDelay)
{
var config = new SmsConfiguration("ACtestaccountsid", "+15550000000")
{
MaxRetries = maxRetries,
RetryDelay = retryDelay,
};
_notificationRepository.GetSmsConfigurationAsync(Arg.Any<CancellationToken>())
.Returns(config);
}
[Fact]
public void SmsNotification_TransientFailure_UsesSmsRetryPolicy()
{
// Task 18: an SMS-typed notification resolves its retry policy from the SMS
// configuration, NOT the shared SMTP policy. The SMTP policy (MaxRetries=10)
// would keep this row Retrying; the SMS policy (MaxRetries=1) parks it after
// this attempt (RetryCount 1 -> 2 >= 1). A pre-fix dispatcher (SMTP-for-all)
// would leave it Retrying with RetryCount=2 — the assertion below fails.
SetupSmtpRetryPolicy(maxRetries: 10, retryDelay: TimeSpan.FromMinutes(1));
SetupSmsRetryPolicy(maxRetries: 1, retryDelay: TimeSpan.FromMinutes(5));
var notification = MakeNotification(type: NotificationType.Sms, retryCount: 1);
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Transient("twilio 503"),
type: NotificationType.Sms);
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is<Notification>(n =>
n.Status == NotificationStatus.Parked &&
n.RetryCount == 2 &&
n.LastError == "twilio 503"),
Arg.Any<CancellationToken>());
});
}
[Fact]
public void EmailNotification_StillUsesSmtpPolicy()
{
// Task 18: an Email row continues to resolve against the SMTP policy even when a
// (tighter) SMS policy is present. SMTP MaxRetries=10 keeps it Retrying
// (RetryCount 1 -> 2 < 10) and schedules the next attempt with the SMTP delay.
SetupSmtpRetryPolicy(maxRetries: 10, retryDelay: TimeSpan.FromMinutes(3));
SetupSmsRetryPolicy(maxRetries: 1, retryDelay: TimeSpan.FromMinutes(5));
var before = DateTimeOffset.UtcNow;
var notification = MakeNotification(type: NotificationType.Email, retryCount: 1);
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"),
type: NotificationType.Email);
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is<Notification>(n =>
n.Status == NotificationStatus.Retrying &&
n.RetryCount == 2 &&
n.NextAttemptAt != null &&
n.NextAttemptAt > before + TimeSpan.FromMinutes(2) &&
n.LastError == "smtp timeout"),
Arg.Any<CancellationToken>());
});
}
}
@@ -169,6 +169,48 @@ public class OAuth2TokenServiceTests
Assert.Equal(1, handler.CallCount);
}
// ── Task 19: configurable OAuth2 authority + scope per SMTP configuration ──
[Fact]
public async Task GetTokenAsync_CustomAuthorityAndScope_PostsToConfiguredEndpoint()
{
// A non-M365 IdP: the stored authority template carries a {tenant} placeholder
// and the scope is provider-specific. Both must flow verbatim into the request.
var handler = new CapturingHttpMessageHandler(
TokenJson("custom-token", expiresIn: 3600));
var client = new HttpClient(handler);
var factory = CreateMockFactory(client);
var service = new OAuth2TokenService(factory, NullLogger<OAuth2TokenService>.Instance);
var token = await service.GetTokenAsync(
"T1:client:secret",
authority: "https://idp.example/{tenant}/token",
scope: "smtp.send");
Assert.Equal("custom-token", token);
Assert.Equal("https://idp.example/T1/token", handler.LastRequestUri!.ToString());
Assert.Equal("smtp.send", handler.LastForm!["scope"]);
}
[Fact]
public async Task GetTokenAsync_DefaultsPreserved_M365()
{
// No authority/scope → today's hardcoded Microsoft 365 endpoint + scope.
var handler = new CapturingHttpMessageHandler(
TokenJson("m365-token", expiresIn: 3600));
var client = new HttpClient(handler);
var factory = CreateMockFactory(client);
var service = new OAuth2TokenService(factory, NullLogger<OAuth2TokenService>.Instance);
var token = await service.GetTokenAsync("tenant:client:secret");
Assert.Equal("m365-token", token);
Assert.Equal(
"https://login.microsoftonline.com/tenant/oauth2/v2.0/token",
handler.LastRequestUri!.ToString());
Assert.Equal("https://outlook.office365.com/.default", handler.LastForm!["scope"]);
}
private static string TokenJson(string accessToken, int expiresIn) =>
JsonSerializer.Serialize(new
{
@@ -250,6 +292,38 @@ public class OAuth2TokenServiceTests
}
}
/// <summary>
/// HTTP handler that captures the request URI and the posted form fields so a
/// test can assert the token endpoint and scope actually sent.
/// </summary>
private class CapturingHttpMessageHandler : HttpMessageHandler
{
private readonly string _response;
public Uri? LastRequestUri { get; private set; }
public IReadOnlyDictionary<string, string>? LastForm { get; private set; }
public CapturingHttpMessageHandler(string response) => _response = response;
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request, CancellationToken cancellationToken)
{
LastRequestUri = request.RequestUri;
var body = request.Content == null
? string.Empty
: await request.Content.ReadAsStringAsync(cancellationToken);
LastForm = body
.Split('&', StringSplitOptions.RemoveEmptyEntries)
.Select(pair => pair.Split('=', 2))
.ToDictionary(
p => Uri.UnescapeDataString(p[0]),
p => p.Length > 1 ? Uri.UnescapeDataString(p[1]) : string.Empty);
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(_response)
};
}
}
/// <summary>
/// Simple mock HTTP handler that returns a fixed response.
/// </summary>
@@ -54,6 +54,26 @@ public class SiteRepositoryTests : IDisposable
Assert.Equal("https://api.example.com", all[0].EndpointUrl);
}
/// <summary>
/// ExternalSystemGateway (Timeout): the per-system <c>TimeoutSeconds</c> stored via
/// <see cref="SiteStorageService.StoreExternalSystemAsync"/> must round-trip through the
/// site SQLite store and back out via the repository, so site-side calls honor it.
/// </summary>
[Fact]
public async Task SiteStorage_ExternalSystem_TimeoutSeconds_RoundTrips()
{
var storage = NewStorage();
await storage.InitializeAsync();
await storage.StoreExternalSystemAsync(
"WeatherApi", "https://api.example.com", "ApiKey", "{\"key\":\"x\"}", null, 7);
var repo = new SiteExternalSystemRepository(storage);
var found = await repo.GetExternalSystemByNameAsync("WeatherApi");
Assert.NotNull(found);
Assert.Equal(7, found!.TimeoutSeconds);
}
/// <summary>
/// SiteRuntime-007: the synthetic ID for an external system must be identical when
/// the storage service and repository are re-created (simulating a process restart).