dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'worktree-agent-aaec6546913f0beae' into arch-review-remediation

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 23:51:37 -04:00
parent 09e350ab0d e0e4b24679
commit 11abda1d1f
16 changed files with 1172 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.CLI/Commands/TemplateCommands.cs
+9 -3
View File
@@ -159,14 +159,20 @@ public static class TemplateCommands
private static Command BuildList(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
{
// The server projects this listing in the database and returns
// TemplateSummary rows (counts, no child collections), so --detail cannot
// produce full definitions here — it only skips the CLI-side column
// projection and renders the summary payload verbatim. Full definitions
// come from `template get --id`.
var detailOption = new Option<bool>("--detail")
{
Description = "Include full template definitions (all attributes/alarms/scripts) in table output. "
+ "Without it, table output is a compact summary (counts only). JSON output is always full."
Description = "Render the raw list payload in table output instead of the compact column projection. "
+ "The server returns summary rows (member counts only) for a listing, so this does NOT include "
+ "attribute/alarm/script definitions — use 'template get --id <int>' for those. No effect on JSON output."
};
var skipOption = new Option<int>("--skip") { Description = "Offset paging: number of items to skip (default 0)" };
var takeOption = new Option<int?>("--take") { Description = "Offset paging: max items to return (1..1000; omit for unlimited)" };
var cmd = new Command("list") { Description = "List all templates (compact table summary; use --detail for the full dump)" };
var cmd = new Command("list") { Description = "List all templates (compact table summary with member counts; definitions via 'template get')" };
cmd.Add(detailOption);
cmd.Add(skipOption);
cmd.Add(takeOption);
src/ZB.MOM.WW.ScadaBridge.CLI/Commands/TemplateTableProjection.cs
+50 -13
View File
@@ -5,12 +5,30 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
/// <summary>
/// Compact table projection for <c>template list</c> / <c>template get</c>.
/// The management API returns full <c>Template</c> entities — every attribute, alarm,
///
/// <para>
/// <c>template get</c> returns a full <c>Template</c> entity — every attribute, alarm,
/// script, and composition inline — which the generic table renderer dumps as one giant
/// cell per template (~171 KB for a real catalogue, unusable in a terminal). This
/// projector reduces each template to id / name / description / parent / derived plus
/// member <em>counts</em>, leaving JSON output untouched (callers pass this only on the
/// table path) and the full dump available via the command's <c>--detail</c> flag.
/// cell per template (~171 KB for a real catalogue, unusable in a terminal).
/// <c>template list</c> no longer returns entities at all: the management
/// <c>ListTemplates</c> handler projects in the DATABASE and returns
/// <c>TemplateSummary</c> rows whose children are already reduced to
/// <c>attributeCount</c>/<c>alarmCount</c>/… scalars.
/// </para>
///
/// <para>
/// This projector handles BOTH shapes: it prefers the pre-computed <c>*Count</c>
/// scalar and falls back to the length of the matching child array, so the same
/// column set renders for a summary row and for a full entity. Reading only the
/// arrays (as it originally did) made <c>template list</c> print a wall of zeros
/// once the server switched to summaries.
/// </para>
///
/// <para>
/// JSON output is left untouched (callers pass this only on the table path), and
/// the command's <c>--detail</c> flag skips the projection to render the raw
/// payload as-is.
/// </para>
/// </summary>
internal static class TemplateTableProjection
{
@@ -67,11 +85,11 @@ internal static class TemplateTableProjection
["description"] = Str(element, "description"),
["parentTemplateId"] = Int(element, "parentTemplateId"),
["isDerived"] = Bool(element, "isDerived"),
["#attrs"] = Count(element, "attributes"),
["#alarms"] = Count(element, "alarms"),
["#scripts"] = Count(element, "scripts"),
["#comps"] = Count(element, "compositions"),
["#nativeAlarms"] = Count(element, "nativeAlarmSources"),
["#attrs"] = Count(element, "attributeCount", "attributes"),
["#alarms"] = Count(element, "alarmCount", "alarms"),
["#scripts"] = Count(element, "scriptCount", "scripts"),
["#comps"] = Count(element, "compositionCount", "compositions"),
["#nativeAlarms"] = Count(element, "nativeAlarmSourceCount", "nativeAlarmSources"),
};
}
@@ -104,9 +122,28 @@ internal static class TemplateTableProjection
? JsonValue.Create(v.GetBoolean())
: null;
private static JsonNode Count(JsonElement obj, string name)
=> JsonValue.Create(
TryGetPropertyCI(obj, name, out var v) && v.ValueKind == JsonValueKind.Array
/// <summary>
/// Member count for a column, preferring the summary payload's pre-computed
/// scalar (<paramref name="countName"/>) and falling back to the length of the
/// full entity's child array (<paramref name="arrayName"/>). Zero when neither
/// is present.
/// </summary>
/// <param name="obj">The template object being projected.</param>
/// <param name="countName">Scalar count property on a <c>TemplateSummary</c> row.</param>
/// <param name="arrayName">Child-collection property on a full <c>Template</c> entity.</param>
/// <returns>The member count as a JSON number.</returns>
private static JsonNode Count(JsonElement obj, string countName, string arrayName)
{
if (TryGetPropertyCI(obj, countName, out var scalar)
&& scalar.ValueKind == JsonValueKind.Number
&& scalar.TryGetInt32(out var n))
{
return JsonValue.Create(n);
}
return JsonValue.Create(
TryGetPropertyCI(obj, arrayName, out var v) && v.ValueKind == JsonValueKind.Array
? v.GetArrayLength()
: 0);
}
}
src/ZB.MOM.WW.ScadaBridge.CLI/ManagementHttpClient.cs
+51 -25
View File
@@ -9,33 +9,52 @@ public class ManagementHttpClient : IDisposable
private readonly HttpClient _httpClient;
/// <summary>
/// WP2.6e (arch-review misc — CLI HttpClient timeout): default overall
/// <see cref="HttpClient.Timeout"/> for the shared client construction (30 s). This
/// bounds a hung/black-holed connection — before this, the public constructor left
/// <see cref="HttpClient.Timeout"/> at its framework default (100 s), silently longer
/// than most CLI callers' own per-request <c>TimeSpan timeout</c> argument
/// (<see cref="SendCommandAsync"/>/<see cref="SendGetAsync"/>/<see cref="SendPostAsync"/>
/// already bound each call via their own <see cref="CancellationTokenSource"/>, but a
/// connection attempt that never completes at all — no response headers, ever — is
/// bounded by <see cref="HttpClient.Timeout"/> instead, since that governs the whole
/// request/response including connect). Config-overridable via the
/// <c>SCADABRIDGE_HTTP_TIMEOUT_SECONDS</c> environment variable, consistent with how
/// every other CLI setting is overridden (see <see cref="CliConfig"/>) — kept
/// self-contained here (no <see cref="CliConfig"/>/command-file plumbing) since CLI
/// commands are owned by a separate work package this phase.
/// Default bound on the CONNECT phase only (30 s) — how long a TCP/TLS
/// connection attempt to a black-holed management endpoint may hang before the
/// call fails.
///
/// <para>
/// It is deliberately NOT an overall request timeout.
/// <see cref="HttpClient.Timeout"/> caps the whole request/response, so setting
/// it to any fixed value silently truncates every caller whose own per-call
/// <c>TimeSpan timeout</c> argument is longer: the effective deadline becomes
/// <c>min(HttpClient.Timeout, caller timeout)</c>. That is exactly what a 30 s
/// client timeout did to <c>deploy site</c>'s 5-minute bulk deploy and to the
/// five-minute <c>bundle</c> export/preview/import calls — each printed a fake
/// <c>504 Request timed out</c> at 30 s while the server carried on working.
/// <see cref="HttpClient.Timeout"/> is therefore
/// <see cref="Timeout.InfiniteTimeSpan"/> and the per-call
/// <see cref="CancellationTokenSource"/> in
/// <see cref="SendCommandAsync"/>/<see cref="SendGetAsync"/>/<see cref="SendPostAsync"/>
/// is the SINGLE overall deadline — it bounds connect too, since the token is
/// passed into the send itself.
/// </para>
///
/// <para>
/// The connect bound lives on <see cref="SocketsHttpHandler.ConnectTimeout"/>
/// instead, which is connect-scoped and so cannot truncate a long-running
/// request that has already reached the server. Overridable via the
/// <c>SCADABRIDGE_HTTP_CONNECT_TIMEOUT_SECONDS</c> environment variable,
/// consistent with how every other CLI setting is overridden (see
/// <see cref="CliConfig"/>) — kept self-contained here (no
/// <see cref="CliConfig"/>/command-file plumbing) since CLI commands are owned
/// by a separate work package this phase.
/// </para>
/// </summary>
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(30);
public static readonly TimeSpan DefaultConnectTimeout = TimeSpan.FromSeconds(30);
/// <summary>Test seam (WP2.6e) — the effective <see cref="HttpClient.Timeout"/> this instance was constructed with.</summary>
/// <summary>Test seam — the effective <see cref="HttpClient.Timeout"/> this instance was constructed with.</summary>
internal TimeSpan EffectiveTimeout { get; }
/// <summary>
/// Resolves the effective default timeout: the <c>SCADABRIDGE_HTTP_TIMEOUT_SECONDS</c>
/// environment variable when set to a positive integer, otherwise <see cref="DefaultTimeout"/>.
/// Resolves the effective connect timeout: the
/// <c>SCADABRIDGE_HTTP_CONNECT_TIMEOUT_SECONDS</c> environment variable when set
/// to a positive integer, otherwise <see cref="DefaultConnectTimeout"/>.
/// </summary>
private static TimeSpan ResolveDefaultTimeout()
/// <returns>The connect timeout to apply to the socket handler.</returns>
internal static TimeSpan ResolveConnectTimeout()
{
var env = Environment.GetEnvironmentVariable("SCADABRIDGE_HTTP_TIMEOUT_SECONDS");
var env = Environment.GetEnvironmentVariable("SCADABRIDGE_HTTP_CONNECT_TIMEOUT_SECONDS");
if (!string.IsNullOrWhiteSpace(env)
&& int.TryParse(env, out var seconds)
&& seconds > 0)
@@ -43,19 +62,26 @@ public class ManagementHttpClient : IDisposable
return TimeSpan.FromSeconds(seconds);
}
return DefaultTimeout;
return DefaultConnectTimeout;
}
/// <summary>
/// Initializes a new instance of the <see cref="ManagementHttpClient"/> class, with
/// <see cref="HttpClient.Timeout"/> set to <see cref="ResolveDefaultTimeout"/>
/// (30 s, or the <c>SCADABRIDGE_HTTP_TIMEOUT_SECONDS</c> override).
/// Initializes a new instance of the <see cref="ManagementHttpClient"/> class with
/// an INFINITE <see cref="HttpClient.Timeout"/> (each call supplies its own
/// deadline) over a <see cref="SocketsHttpHandler"/> whose
/// <see cref="SocketsHttpHandler.ConnectTimeout"/> is
/// <see cref="ResolveConnectTimeout"/>.
/// </summary>
/// <param name="baseUrl">The base URL for the management API.</param>
/// <param name="username">The username for HTTP Basic authentication.</param>
/// <param name="password">The password for HTTP Basic authentication.</param>
public ManagementHttpClient(string baseUrl, string username, string password)
: this(new HttpClient { Timeout = ResolveDefaultTimeout() }, baseUrl, username, password)
: this(
new HttpClient(new SocketsHttpHandler { ConnectTimeout = ResolveConnectTimeout() })
{
Timeout = Timeout.InfiniteTimeSpan
},
baseUrl, username, password)
{
}
src/ZB.MOM.WW.ScadaBridge.CLI/README.md
+16 -11
View File
@@ -86,28 +86,33 @@ Exit codes:
#### `template list`
List all templates. **Table** output (`--format table`) shows a compact summary — id,
name, description, parent, and member **counts** (`#attrs`, `#alarms`, `#scripts`,
`#comps`, `#nativeAlarms`) — so it stays readable in a terminal. Add `--detail` to dump
the full attribute/alarm/script/composition definitions in the table. **JSON** output
(`--format json`) is always the full, unmodified payload regardless of `--detail`.
List all templates. The server projects this listing in the database and returns
**summary rows** — no child collections, member counts only. **Table** output
(`--format table`) renders them as id, name, description, parent, and member **counts**
(`#attrs`, `#alarms`, `#scripts`, `#comps`, `#nativeAlarms`), so it stays readable in a
terminal. `--detail` skips that column projection and renders the summary payload
verbatim; it does **not** add attribute/alarm/script definitions — a listing never
carries them. Use `template get --id <int>` for a template's full definition. **JSON**
output (`--format json`) is always the unmodified payload regardless of `--detail`.
```sh
scadabridge --url <url> template list # compact table
scadabridge --url <url> --format table template list --detail # full table dump
scadabridge --url <url> --format json template list # full JSON (always)
scadabridge --url <url> template list # compact table
scadabridge --url <url> --format table template list --detail # raw summary rows
scadabridge --url <url> --format json template list # unmodified JSON (always)
```
| Option | Required | Description |
|--------|----------|-------------|
| `--detail` | no | Include full definitions in table output (no effect on JSON) |
| `--detail` | no | Render the raw summary payload instead of the compact column projection (no effect on JSON; does not add definitions) |
| `--skip` | no | Offset paging: number of items to skip (default 0) |
| `--take` | no | Offset paging: max items to return (clamped 1..1000; omit for unlimited) |
#### `template get`
Get a single template by ID. Like `template list`, table output is a compact summary
unless `--detail` is supplied; JSON output is always full.
Get a single template by ID. Unlike `template list`, this returns the **full** template
entity with every attribute, alarm, script and composition inline. Table output is a
compact summary (the same counts as `template list`) unless `--detail` is supplied, which
dumps the full definitions; JSON output is always full.
```sh
scadabridge --url <url> template get --id <int> [--detail]
src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/SiteRepository.cs
+62 -2
View File
@@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
@@ -11,14 +12,23 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
public class SiteRepository : ISiteRepository
{
private readonly ScadaBridgeDbContext _dbContext;
private readonly ITemplateGraphWatermark? _watermark;
/// <summary>
/// Initializes a new instance of the SiteRepository.
/// </summary>
/// <param name="dbContext">The database context.</param>
public SiteRepository(ScadaBridgeDbContext dbContext)
/// <param name="watermark">
/// Graph version watermark bumped when a committed change alters a flattening
/// input this repository owns (see <see cref="SaveChangesAsync"/>). Optional so
/// existing single-argument construction (tests, ad-hoc tooling) keeps
/// compiling; when absent the caches simply never get the invalidation signal
/// from this repository, which is the pre-existing behaviour.
/// </param>
public SiteRepository(ScadaBridgeDbContext dbContext, ITemplateGraphWatermark? watermark = null)
{
_dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext));
_watermark = watermark;
}
// --- Sites ---
@@ -166,8 +176,58 @@ public class SiteRepository : ISiteRepository
}
/// <inheritdoc />
/// <remarks>
/// Bumps the <see cref="ITemplateGraphWatermark"/> after a successful commit
/// that touched a <see cref="DataConnection"/>.
///
/// <para>
/// A data connection's <c>Protocol</c>, primary/backup configuration and
/// failover retry count are flattening inputs — <c>FlatteningService</c>
/// packages them into the flattened config's <c>Connections</c> map and
/// <c>RevisionHashService</c> folds them into the revision hash. Without a bump,
/// the process-wide <c>StaleInstanceProbe</c> memo and the flatten-session
/// caches keep reporting the pre-edit hash for every instance bound to the
/// connection, so an instance whose deployed config genuinely drifted reads as
/// up to date.
/// </para>
///
/// <para>
/// The bump is <see cref="ITemplateGraphWatermark.BumpAll"/> rather than a
/// per-template one because a connection has no owning template: the affected
/// set is "every instance at the site bound to it", which is exactly what the
/// unattributed fallback is for. Applied only AFTER a successful commit, so a
/// failed save does not invalidate caches for a change that never landed.
/// Sites, areas and instances saved here are left alone — only the connection
/// rows feed the flattener.
/// </para>
/// </remarks>
public async Task<int> SaveChangesAsync(CancellationToken cancellationToken = default)
{
return await _dbContext.SaveChangesAsync(cancellationToken);
// Inspect BEFORE committing: afterwards every entry is Unchanged and the
// attribution is lost.
var connectionsChanged = _watermark is not null && HasPendingDataConnectionChange();
var written = await _dbContext.SaveChangesAsync(cancellationToken);
if (connectionsChanged)
_watermark!.BumpAll();
return written;
}
/// <summary>
/// True when the change tracker holds an added, modified or deleted
/// <see cref="DataConnection"/>.
/// </summary>
/// <returns>Whether a data-connection mutation is about to be committed.</returns>
private bool HasPendingDataConnectionChange()
{
foreach (var entry in _dbContext.ChangeTracker.Entries<DataConnection>())
{
if (entry.State is EntityState.Added or EntityState.Modified or EntityState.Deleted)
return true;
}
return false;
}
}
src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/TemplateEngineRepository.cs
+12 -1
View File
@@ -865,7 +865,18 @@ public class TemplateEngineRepository : ITemplateEngineRepository
unattributed = true;
break;
default:
// Areas, folders and anything else do not feed the flattener.
// Anything this switch does not recognise is invalidated
// conservatively, as the doc comment above promises. The
// previous "areas, folders and anything else do not feed the
// flattener" no-op was wrong in at least one shipped case:
// DataConnection rows land here, and their Protocol /
// Primary+Backup configuration / FailoverRetryCount ARE hash
// inputs (FlatteningService packages them as ConnectionConfig,
// RevisionHashService hashes them as HashableConnection). A
// silent no-op on an unrecognised entity is the one failure
// mode this watermark must not have: it produces STALE work,
// whereas an over-broad bump only ever produces extra work.
unattributed = true;
break;
}
}
src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentService.cs
+301 -89
View File
@@ -193,7 +193,8 @@ public class DeploymentService
using (prepared.LockHandle)
{
var outcome = await SendDeploymentAsync(prepared, cancellationToken);
var outcome = await StageAndSendDeploymentAsync(
prepared, user, dbGate: null, cancellationToken);
return await FinalizeDeploymentAsync(prepared, outcome, user, cancellationToken);
}
}
@@ -201,9 +202,18 @@ public class DeploymentService
/// <summary>
/// Phase 1 of a deployment (serial, database-bound): validate the state
/// transition, take the per-instance operation lock, mint the deployment id,
/// flatten + validate, run query-before-redeploy reconciliation, stage the
/// <c>PendingDeployment</c> row and insert the <c>InProgress</c>
/// <see cref="DeploymentRecord"/>.
/// flatten + validate, run query-before-redeploy reconciliation and insert the
/// <c>InProgress</c> <see cref="DeploymentRecord"/>.
///
/// <para>
/// Note what is NOT here: the <c>PendingDeployment</c> row. Its fetch token
/// expires <see cref="CommunicationOptions.PendingDeploymentTtl"/> after it is
/// STAGED, so staging in phase 1 started the clock on every instance in a bulk
/// deploy at once while phase 2 reached them one batch at a time — with enough
/// instances the tail's tokens expired before their command was ever sent, and
/// the site's fetch 404'd. Staging therefore moved to
/// <see cref="StageAndSendDeploymentAsync"/>, immediately before each send.
/// </para>
///
/// <para>
/// Everything here touches the scoped, non-thread-safe <c>DbContext</c>, so it
@@ -367,48 +377,135 @@ public class DeploymentService
try
{
// Notify-and-fetch: instead of shipping the (potentially oversized,
// silently-dropped >128 KB) flattened config inline in a
// DeployInstanceCommand, stage it in a PendingDeployment row and send
// a small RefreshDeploymentCommand. The site fetches the config from
// CentralFetchBaseUrl over HTTP using the per-deployment fetch token.
// Site routing is resolved here (a repository read) so phase 2 needs
// nothing but the staging write itself.
var siteId = await ResolveSiteIdentifierAsync(instance.SiteId, cancellationToken);
var token = DeploymentFetchToken.Generate();
var stagedAt = DateTimeOffset.UtcNow;
await _repository.AddPendingDeploymentAsync(new PendingDeployment(
deploymentId, instanceId, revisionHash, configJson, token,
stagedAt, stagedAt + _commOptions.PendingDeploymentTtl), cancellationToken);
await _repository.SaveChangesAsync(cancellationToken);
var command = new RefreshDeploymentCommand(
deploymentId, instance.UniqueName, revisionHash, user, stagedAt,
_commOptions.CentralFetchBaseUrl, token);
// Cleanup of the staged PendingDeployment is TTL-based ONLY — the row
// is deliberately NOT deleted on success or on failure. On a
// central-side Ask timeout the site may have applied AND told the
// standby node to fetch; deleting now would 404 that in-flight
// standby fetch and break failover. Supersession bounds pending rows
// to ≤1 per instance and the fetch endpoint enforces the TTL, so
// leaving rows for TTL purge is safe. Expired rows are swept by the
// central PendingDeploymentPurgeActor singleton on its
// CommunicationOptions.PendingDeploymentPurgeInterval cadence.
return new PreparedDeployment(
instance, deploymentId, record, revisionHash, configJson,
siteId, command, lockHandle, EarlyResult: null);
siteId, lockHandle, EarlyResult: null);
}
catch (Exception ex)
{
// Staging failed before anything was sent. Record the failure exactly
// as the post-send path does (never leave the record InProgress) and
// release the lock — there is no phase 2 or 3 to run.
// Preparation failed before anything was staged or sent. Record the
// failure exactly as the post-send path does (never leave the record
// InProgress) and release the lock — there is no phase 2 or 3 to run.
await MarkDeploymentFailedAsync(record, instance, deploymentId, user, ex);
lockHandle.Dispose();
return PreparedDeployment.Resolved(FailureResultFor(ex));
}
}
/// <summary>
/// Stages the deployment's <c>PendingDeployment</c> row and immediately sends
/// its <c>RefreshDeploymentCommand</c> — phase 2.
///
/// <para>
/// Notify-and-fetch: instead of shipping the (potentially oversized,
/// silently-dropped &gt;128 KB) flattened config inline in a
/// <c>DeployInstanceCommand</c>, the config is staged in a
/// <c>PendingDeployment</c> row and a small <c>RefreshDeploymentCommand</c> is
/// sent; the site then fetches the config from <c>CentralFetchBaseUrl</c> over
/// HTTP using the per-deployment fetch token.
/// </para>
///
/// <para>
/// <b>Staging is immediately before the send, on purpose.</b> The row's
/// <c>ExpiresAt</c> is <c>stagedAt + PendingDeploymentTtl</c> (5 min by
/// default) and the fetch endpoint enforces it, so the gap between staging and
/// sending is dead time burned off the token's life. Staging the whole batch up
/// front in phase 1 made that gap grow with batch size — for a large site the
/// last instances' tokens could expire before their command was even sent.
/// </para>
///
/// <para>
/// <b>Serialisation.</b> Staging is the ONE repository touch in this otherwise
/// network-only phase, and the scoped <c>DbContext</c> is not thread-safe, so a
/// bulk deploy passes <paramref name="dbGate"/> — a 1-permit semaphore that lets
/// exactly one in-flight send do its staging write at a time. The sends
/// themselves stay fully concurrent (they touch no repository), so this costs no
/// parallelism, and the single-instance path passes <see langword="null"/>
/// because there is nothing to serialise against.
/// </para>
///
/// <para>
/// A staging failure is returned as the phase-2 fault rather than thrown, so
/// phase 3 marks the record Failed on exactly the same path as a send fault —
/// the record must never be left InProgress.
/// </para>
/// </summary>
/// <param name="prepared">The prepared deployment to stage and send.</param>
/// <param name="user">User attributed on the refresh command.</param>
/// <param name="dbGate">Serialises the staging write across a concurrent batch; <see langword="null"/> for a batch of one.</param>
/// <param name="cancellationToken">Cancellation token, already carrying any per-instance deadline.</param>
/// <returns>The site's response, or the exception that prevented one.</returns>
private async Task<SendOutcome> StageAndSendDeploymentAsync(
PreparedDeployment prepared,
string user,
SemaphoreSlim? dbGate,
CancellationToken cancellationToken)
{
RefreshDeploymentCommand command;
try
{
if (dbGate is not null)
await dbGate.WaitAsync(cancellationToken);
try
{
command = await StagePendingDeploymentAsync(prepared, user, cancellationToken);
}
finally
{
dbGate?.Release();
}
}
catch (Exception ex)
{
return new SendOutcome(null, ex);
}
return await SendDeploymentAsync(prepared, command, cancellationToken);
}
/// <summary>
/// Writes the <c>PendingDeployment</c> row for a prepared deployment and builds
/// the matching <c>RefreshDeploymentCommand</c>, stamping <c>stagedAt</c> at the
/// moment of the write.
///
/// <para>
/// Cleanup of the staged row is TTL-based ONLY — it is deliberately NOT deleted
/// on success or on failure. On a central-side Ask timeout the site may have
/// applied AND told the standby node to fetch; deleting now would 404 that
/// in-flight standby fetch and break failover. Supersession bounds pending rows
/// to ≤1 per instance and the fetch endpoint enforces the TTL, so leaving rows
/// for TTL purge is safe. Expired rows are swept by the central
/// <c>PendingDeploymentPurgeActor</c> singleton on its
/// <see cref="CommunicationOptions.PendingDeploymentPurgeInterval"/> cadence.
/// </para>
/// </summary>
/// <param name="prepared">The prepared deployment being staged.</param>
/// <param name="user">User attributed on the refresh command.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The refresh command to send to the site.</returns>
private async Task<RefreshDeploymentCommand> StagePendingDeploymentAsync(
PreparedDeployment prepared,
string user,
CancellationToken cancellationToken)
{
var token = DeploymentFetchToken.Generate();
var stagedAt = DateTimeOffset.UtcNow;
await _repository.AddPendingDeploymentAsync(new PendingDeployment(
prepared.DeploymentId, prepared.Instance.Id, prepared.RevisionHash,
prepared.ConfigJson, token,
stagedAt, stagedAt + _commOptions.PendingDeploymentTtl), cancellationToken);
await _repository.SaveChangesAsync(cancellationToken);
return new RefreshDeploymentCommand(
prepared.DeploymentId, prepared.Instance.UniqueName, prepared.RevisionHash,
user, stagedAt, _commOptions.CentralFetchBaseUrl, token);
}
/// <summary>
/// Phase 2 of a deployment (parallelisable, network-bound): the
/// <c>RefreshDeploymentCommand</c> round-trip to the site. Touches no
@@ -416,10 +513,12 @@ public class DeploymentService
/// out across a batch while phases 1 and 3 stay serial.
/// </summary>
/// <param name="prepared">The prepared deployment to send.</param>
/// <param name="command">The refresh command built by the staging step immediately before this call.</param>
/// <param name="cancellationToken">Cancellation token, already carrying any per-instance deadline.</param>
/// <returns>The site's response, or the exception that prevented one.</returns>
private async Task<SendOutcome> SendDeploymentAsync(
PreparedDeployment prepared,
RefreshDeploymentCommand command,
CancellationToken cancellationToken)
{
try
@@ -429,7 +528,7 @@ public class DeploymentService
prepared.DeploymentId, prepared.Instance.UniqueName, prepared.SiteIdentifier);
var response = await _communicationService.RefreshDeploymentAsync(
prepared.SiteIdentifier, prepared.Command, cancellationToken);
prepared.SiteIdentifier, command, cancellationToken);
return new SendOutcome(response, null);
}
catch (Exception ex)
@@ -630,7 +729,6 @@ public class DeploymentService
string RevisionHash,
string ConfigJson,
string SiteIdentifier,
RefreshDeploymentCommand Command,
IDisposable? LockHandle,
Result<DeploymentRecord>? EarlyResult)
{
@@ -638,7 +736,7 @@ public class DeploymentService
/// <param name="result">The result to hand back to the caller.</param>
/// <returns>A prepared deployment whose <see cref="EarlyResult"/> is set.</returns>
public static PreparedDeployment Resolved(Result<DeploymentRecord> result) =>
new(null!, string.Empty, null!, string.Empty, string.Empty, string.Empty, null!, null, result);
new(null!, string.Empty, null!, string.Empty, string.Empty, string.Empty, null, result);
}
/// <summary>
@@ -655,21 +753,23 @@ public class DeploymentService
/// </para>
/// <list type="number">
/// <item>
/// <b>Prepare (serial).</b> Every instance is flattened, validated, staged
/// and given an <c>InProgress</c> record on the caller's single scoped
/// <b>Prepare (serial).</b> Every instance is flattened, validated and given
/// an <c>InProgress</c> record on the caller's single scoped
/// <c>DbContext</c>. All instances share ONE <see cref="FlattenSession"/>,
/// so a template chain common to N instances is walked once, and the
/// session-global queries (shared scripts, schema library, the site's data
/// connections) run once for the whole batch instead of once per instance.
/// </item>
/// <item>
/// <b>Send (bounded parallel).</b> Site round-trips run concurrently up to
/// <see cref="DeploymentManagerOptions.SiteDeploymentMaxParallelism"/>,
/// <b>Stage + send (bounded parallel).</b> Site round-trips run concurrently
/// up to <see cref="DeploymentManagerOptions.SiteDeploymentMaxParallelism"/>,
/// each under its own
/// <see cref="DeploymentManagerOptions.SiteDeploymentTimeoutPerInstance"/>
/// deadline, so one wedged instance cannot stall the batch. This phase
/// touches no repository — that is precisely why it is the only phase that
/// may run in parallel against a non-thread-safe <c>DbContext</c>.
/// deadline, so one wedged instance cannot stall the batch. Each instance's
/// <c>PendingDeployment</c> row is staged here, immediately before its own
/// send, so a fetch token's TTL is not burned waiting for the batch ahead of
/// it; that single repository touch is serialised behind a 1-permit
/// semaphore, since the <c>DbContext</c> is not thread-safe.
/// </item>
/// <item>
/// <b>Finalize (serial).</b> Terminal statuses, post-success side effects
@@ -710,54 +810,81 @@ public class DeploymentService
var prepared = new List<PreparedDeployment>(instances.Count);
var results = new List<InstanceDeploymentResult>(instances.Count);
foreach (var instance in instances)
// Index of the first prepared deployment phase 3 has NOT yet finalised.
// Everything from here to the end of `prepared` still owns an operation
// lock and an InProgress record, so the cleanup handler below knows exactly
// what it must unwind if the operation escapes early.
var finalizedUpTo = 0;
try
{
cancellationToken.ThrowIfCancellationRequested();
foreach (var instance in instances)
{
cancellationToken.ThrowIfCancellationRequested();
PreparedDeployment step;
try
{
step = await PrepareDeploymentAsync(instance.Id, user, session, cancellationToken);
}
catch (Exception ex)
{
// A prepare fault (most commonly a TimeoutException from the
// operation lock because another operation holds this instance)
// fails only this instance. Recording it and moving on is what
// makes a bulk deploy usable while individual instances are busy.
_logger.LogWarning(ex,
"Preparing instance {Instance} for bulk deployment of site {SiteId} failed",
instance.UniqueName, site.SiteIdentifier);
results.Add(new InstanceDeploymentResult(
instance.Id, instance.UniqueName, null, false, ex.Message));
continue;
PreparedDeployment step;
try
{
step = await PrepareDeploymentAsync(instance.Id, user, session, cancellationToken);
}
catch (Exception ex)
{
// A prepare fault (most commonly a TimeoutException from the
// operation lock because another operation holds this instance)
// fails only this instance. Recording it and moving on is what
// makes a bulk deploy usable while individual instances are busy.
_logger.LogWarning(ex,
"Preparing instance {Instance} for bulk deployment of site {SiteId} failed",
instance.UniqueName, site.SiteIdentifier);
results.Add(new InstanceDeploymentResult(
instance.Id, instance.UniqueName, null, false, ex.Message));
continue;
}
if (step.EarlyResult is { } early)
{
results.Add(ToInstanceResult(instance.Id, instance.UniqueName, early));
continue;
}
prepared.Add(step);
}
if (step.EarlyResult is { } early)
{
results.Add(ToInstanceResult(instance.Id, instance.UniqueName, early));
continue;
}
// ---- Phase 2: stage + bounded-parallel site round-trips. ----
// Never throws: faults (cancellation included) come back as per-instance
// outcomes so phase 3 still runs for every prepared deployment.
var outcomes = await SendPreparedAsync(prepared, user, cancellationToken);
prepared.Add(step);
// ---- Phase 3: finalize, serially, releasing each lock as we go. ----
for (; finalizedUpTo < prepared.Count; finalizedUpTo++)
{
var step = prepared[finalizedUpTo];
try
{
var result = await FinalizeDeploymentAsync(step, outcomes[finalizedUpTo], user, cancellationToken);
results.Add(ToInstanceResult(step.Instance.Id, step.Instance.UniqueName, result, step.DeploymentId));
}
finally
{
step.LockHandle?.Dispose();
}
}
}
// ---- Phase 2: bounded-parallel site round-trips. ----
var outcomes = await SendPreparedAsync(prepared, cancellationToken);
// ---- Phase 3: finalize, serially, releasing each lock as we go. ----
for (var i = 0; i < prepared.Count; i++)
catch (Exception ex)
{
var step = prepared[i];
try
{
var result = await FinalizeDeploymentAsync(step, outcomes[i], user, cancellationToken);
results.Add(ToInstanceResult(step.Instance.Id, step.Instance.UniqueName, result, step.DeploymentId));
}
finally
{
step.LockHandle?.Dispose();
}
// Nothing above may escape while a prepared deployment still holds its
// per-instance operation lock: OperationLockManager hands out real
// semaphores, so an undisposed handle wedges that instance against
// EVERY future mutating command for the life of the process — a
// permanent, restart-only outage for one instance. The two live escape
// routes are phase 1's ThrowIfCancellationRequested and a fault from
// phase 3's own persistence; both land here.
//
// Records are finalised as Failed for the same reason the send/persist
// paths do it: a deployment must never be left InProgress. Uses
// CancellationToken.None inside (see MarkDeploymentFailedAsync) so the
// already-cancelled token cannot abort the cleanup writes themselves.
await ReleasePreparedAsync(prepared, finalizedUpTo, user, ex);
throw;
}
var summary = new SiteDeploymentSummary(
@@ -776,15 +903,76 @@ public class DeploymentService
}
/// <summary>
/// Runs phase 2 for a whole batch: every prepared deployment's site round-trip,
/// concurrent up to <see cref="DeploymentManagerOptions.SiteDeploymentMaxParallelism"/>
/// and each bounded by
/// Unwinds the prepared deployments a bulk deploy never finalised, after the
/// operation escaped early (cancellation, or a fault in phase 3's own
/// persistence). For each one it writes the terminal Failed status and releases
/// the per-instance operation lock.
///
/// <para>
/// Best-effort throughout: a fault while unwinding one entry must not stop the
/// remaining locks from being released, and must not mask the original
/// exception the caller is about to rethrow. The lock release in particular is
/// in a <c>finally</c>, because leaking it is the unrecoverable outcome — the
/// Failed status can be reconciled by an operator, a wedged semaphore cannot.
/// </para>
/// </summary>
/// <param name="prepared">The batch's prepared deployments.</param>
/// <param name="from">Index of the first entry phase 3 did not finalise.</param>
/// <param name="user">User attributed on the failure audit rows.</param>
/// <param name="cause">The exception that aborted the operation.</param>
/// <returns>A task that completes once every outstanding entry has been unwound.</returns>
private async Task ReleasePreparedAsync(
IReadOnlyList<PreparedDeployment> prepared,
int from,
string user,
Exception cause)
{
for (var i = from; i < prepared.Count; i++)
{
var step = prepared[i];
try
{
await MarkDeploymentFailedAsync(
step.Record, step.Instance, step.DeploymentId, user, cause);
}
catch (Exception ex)
{
_logger.LogError(ex,
"Failed to finalize deployment {DeploymentId} for instance {Instance} while unwinding " +
"an aborted bulk deployment",
step.DeploymentId, step.Instance.UniqueName);
}
finally
{
step.LockHandle?.Dispose();
}
}
}
/// <summary>
/// Runs phase 2 for a whole batch: every prepared deployment's staging write
/// plus site round-trip, concurrent up to
/// <see cref="DeploymentManagerOptions.SiteDeploymentMaxParallelism"/> and each
/// bounded by
/// <see cref="DeploymentManagerOptions.SiteDeploymentTimeoutPerInstance"/>.
/// Outcomes are returned positionally so phase 3 can pair them back with their
/// prepared deployment.
///
/// <para>
/// Never throws on cancellation: the whole point of returning outcomes
/// positionally is that phase 3 must run for EVERY prepared deployment, whatever
/// happened in phase 2 — otherwise records stay InProgress and their operation
/// locks stay held. A cancelled instance is recorded as its own
/// <see cref="SendOutcome"/> fault and finalised as Failed like any other.
/// </para>
/// </summary>
/// <param name="prepared">The prepared deployments to stage and send.</param>
/// <param name="user">User attributed on each refresh command.</param>
/// <param name="cancellationToken">Cancellation token for the batch.</param>
/// <returns>One outcome per prepared deployment, positionally aligned.</returns>
private async Task<SendOutcome[]> SendPreparedAsync(
IReadOnlyList<PreparedDeployment> prepared,
string user,
CancellationToken cancellationToken)
{
var outcomes = new SendOutcome[prepared.Count];
@@ -792,15 +980,39 @@ public class DeploymentService
return outcomes;
using var gate = new SemaphoreSlim(_options.SiteDeploymentMaxParallelism);
// Serialises the ONE repository touch left in this phase (the staging
// write) across the concurrent sends — the scoped DbContext is not
// thread-safe. The sends themselves stay fully concurrent.
using var dbGate = new SemaphoreSlim(1, 1);
var sends = prepared.Select(async (step, index) =>
{
await gate.WaitAsync(cancellationToken);
// Cancellation must not escape as an exception: it would abandon phase 3
// for every instance, leaking their operation locks (a wedged
// per-instance semaphore is permanent for the life of the process) and
// stranding their records InProgress.
try
{
await gate.WaitAsync(cancellationToken);
}
catch (OperationCanceledException ex)
{
outcomes[index] = new SendOutcome(null, ex);
return;
}
try
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cts.CancelAfter(_options.SiteDeploymentTimeoutPerInstance);
outcomes[index] = await SendDeploymentAsync(step, cts.Token);
outcomes[index] = await StageAndSendDeploymentAsync(step, user, dbGate, cts.Token);
}
catch (Exception ex)
{
// StageAndSendDeploymentAsync already converts its own faults into
// outcomes; this is the belt-and-braces net so no send task can
// fault Task.WhenAll and skip phase 3.
outcomes[index] = new SendOutcome(null, ex);
}
finally
{
src/ZB.MOM.WW.ScadaBridge.TemplateEngine/Validation/ScriptCompileVerdictCache.cs
+24 -7
View File
@@ -49,7 +49,12 @@ public static class ScriptCompileVerdictCache
/// <summary>
/// Upper bound on entries in the hot generation. The cache holds at most
/// <c>2 × SegmentCapacity</c> entries in total (hot + cold), preserving the
/// previous 4096-entry ceiling.
/// previous 4096-entry ceiling. Enforced on EVERY insert into hot — fresh
/// verdicts and cold-generation promotions alike, both of which go through
/// <see cref="Store"/>. (Hot can momentarily exceed the bound by the number of
/// threads that pass the pre-rotation check concurrently; the bound is a
/// memory ceiling, not a hard invariant, and that overshoot is bounded by
/// concurrency rather than by working-set size.)
/// </summary>
private const int SegmentCapacity = 2048;
@@ -61,8 +66,13 @@ public static class ScriptCompileVerdictCache
/// </summary>
private static readonly object RotateGate = new();
private static ConcurrentDictionary<string, (bool Ok, string? Error)> _hot = new();
private static ConcurrentDictionary<string, (bool Ok, string? Error)> _cold = new();
// Volatile: both fields are REPLACED wholesale by a rotation under
// RotateGate, while readers on GetOrAdd/Store run lock-free. Without the
// volatile read a reader is free to cache the reference (or observe the
// rotation's two writes out of order) and keep serving — or worse, keep
// inserting into — a generation the rotation has already retired.
private static volatile ConcurrentDictionary<string, (bool Ok, string? Error)> _hot = new();
private static volatile ConcurrentDictionary<string, (bool Ok, string? Error)> _cold = new();
private static long _hits;
private static long _evictions;
@@ -110,11 +120,18 @@ public static class ScriptCompileVerdictCache
if (cold.TryGetValue(key, out verdict))
{
// Promote: the entry is in active use, so it must survive the next
// rotation. Writing to the CURRENT hot generation (re-read, in case a
// rotation happened since the snapshot) is what makes this an LRU
// rather than a fixed-lifetime cache.
// rotation. Goes through Store (rather than a direct `_hot[key] =`)
// so a promotion obeys SegmentCapacity like any other insert — a
// direct write let hot grow past the segment bound whenever a working
// set larger than SegmentCapacity was being re-read, taking the true
// ceiling to 3 × SegmentCapacity against a documented 2 ×. Store
// writes to the CURRENT hot generation (re-read inside, in case a
// rotation happened since the snapshot above), which is what makes
// this an LRU rather than a fixed-lifetime cache; a rotation
// triggered BY this promotion still keeps the entry, because the
// insert lands in the fresh hot generation afterwards.
Interlocked.Increment(ref _hits);
_hot[key] = verdict;
Store(key, verdict);
return verdict;
}
src/ZB.MOM.WW.ScadaBridge.Transport/Import/BundleImporter.cs
+51 -1
View File
@@ -87,6 +87,12 @@ public sealed class BundleImporter : IBundleImporter
// then best-effort empty (informational only, never gates the import).
private readonly IStaleInstanceProbe? _staleInstanceProbe;
private readonly IScriptArtifactChangeBus? _scriptArtifactChangeBus;
// Optional. Bumped once per apply ATTEMPT (see ApplyAsync) because this
// importer commits through the raw DbContext, bypassing
// TemplateEngineRepository.SaveChangesAsync — the only place that otherwise
// derives watermark bumps from the change tracker. Null in hosts that don't
// register it; the caches then simply never get this invalidation signal.
private readonly ITemplateGraphWatermark? _watermark;
private readonly ILogger<BundleImporter>? _logger;
/// <summary>
@@ -112,6 +118,7 @@ public sealed class BundleImporter : IBundleImporter
/// <param name="staleInstanceProbe">Optional: recomputes a deployed instance's current revision hash so the importer can enumerate instances stale-ed by a template overwrite. Null when no flattening pipeline is registered (Transport-without-DeploymentManager hosts) — staleness is then skipped.</param>
/// <param name="scriptArtifactChangeBus">Optional: node-local bus notified AFTER a successful commit that script-bearing artifacts (ApiMethod / SharedScript / Template) were overwritten or renamed, so compiled-handler caches can invalidate. Null on hosts that don't register it (staleness/invalidation then relies on the consumer's own self-heal).</param>
/// <param name="logger">Optional logger.</param>
/// <param name="watermark">Optional: template-graph version watermark, bumped once per apply attempt so the flatten-session caches and the process-wide staleness memo cannot serve pre-import (or rolled-back) readings. Null on hosts that don't register it.</param>
public BundleImporter(
BundleSerializer bundleSerializer,
ManifestValidator manifestValidator,
@@ -132,7 +139,8 @@ public sealed class BundleImporter : IBundleImporter
SemanticValidator semanticValidator,
IStaleInstanceProbe? staleInstanceProbe = null,
IScriptArtifactChangeBus? scriptArtifactChangeBus = null,
ILogger<BundleImporter>? logger = null)
ILogger<BundleImporter>? logger = null,
ITemplateGraphWatermark? watermark = null)
{
_bundleSerializer = bundleSerializer ?? throw new ArgumentNullException(nameof(bundleSerializer));
_manifestValidator = manifestValidator ?? throw new ArgumentNullException(nameof(manifestValidator));
@@ -153,6 +161,7 @@ public sealed class BundleImporter : IBundleImporter
_semanticValidator = semanticValidator ?? throw new ArgumentNullException(nameof(semanticValidator));
_staleInstanceProbe = staleInstanceProbe;
_scriptArtifactChangeBus = scriptArtifactChangeBus;
_watermark = watermark;
_logger = logger;
}
@@ -1327,6 +1336,7 @@ public sealed class BundleImporter : IBundleImporter
content, resolutions, nameMap, session, bundleImportId, user, ct).ConfigureAwait(false);
await tx.CommitAsync(ct).ConfigureAwait(false);
await tx.DisposeAsync().ConfigureAwait(false);
BumpGraphWatermark();
return applied;
}
catch
@@ -1351,6 +1361,14 @@ public sealed class BundleImporter : IBundleImporter
// persist on the next SaveChangesAsync (a retry or the
// failure-row write below).
_dbContext.ChangeTracker.Clear();
// Bump on the ROLLBACK path too. ComputeStaleInstanceIdsAsync
// runs PRE-commit (it reads the staged change tracker), so a
// rolled-back attempt has already written StaleInstanceProbe
// memos describing state that never landed. Leaving the
// watermark untouched would keep those memos "current"
// forever — including for the strategy's own next retry.
BumpGraphWatermark();
throw;
}
}).ConfigureAwait(false);
@@ -1651,6 +1669,38 @@ public sealed class BundleImporter : IBundleImporter
Warnings: validationWarnings);
}
/// <summary>
/// Invalidates every template-graph cache keyed on the
/// <see cref="ITemplateGraphWatermark"/>, called once per apply ATTEMPT — after
/// the commit on the success path, and after the rollback on the failure path.
///
/// <para>
/// <b>Why the importer must do this itself.</b> Watermark bumps are normally
/// derived from the change tracker by
/// <c>TemplateEngineRepository.SaveChangesAsync</c>, but this importer commits
/// through the raw <c>ScadaBridgeDbContext</c> (a single deferred
/// <c>SaveChangesAsync</c> plus a handful of intra-transaction flushes), so not
/// one of its writes went through that path. The observable bug: a second
/// import overwriting the same template could be OMITTED from
/// <c>ImportResult.StaleInstanceIds</c>, because <c>StaleInstanceProbe</c>'s
/// process-static memo — validated purely against watermark readings that never
/// moved — served the FIRST import's revision hash.
/// </para>
///
/// <para>
/// <b>Why both paths, and why <see cref="ITemplateGraphWatermark.BumpAll"/>.</b>
/// The probe runs pre-commit, so a rolled-back attempt leaves memos for state
/// that never landed; bumping on rollback is what makes the memo non-transactional
/// yet still sound (the simplest correct shape — a transactional memo would mean
/// threading commit/rollback awareness through a process-static cache). And an
/// import touches an arbitrary, unattributed set of templates, instances and data
/// connections, so the unattributed global bump is both the correct granularity
/// and cheap: an import is a rare, heavyweight operation, and the watermark can
/// only ever cause EXTRA flattening work, never stale work.
/// </para>
/// </summary>
private void BumpGraphWatermark() => _watermark?.BumpAll();
/// <summary>
/// T-007: zeros the session's <see cref="BundleSession.DecryptedContent"/>
/// buffer in place so any caller still holding a reference observes the