merge: external-system retry policy end-to-end (register row 27) — MaxRetries/RetryDelay now travel artifact→gRPC→site schema→S&F enqueue, CLI --max-retries/--retry-delay-seconds, ArtifactDiff compares them

This commit is contained in:
Joseph Doherty
2026-08-15 17:56:21 -04:00
28 changed files with 1036 additions and 453 deletions
@@ -50,6 +50,9 @@ public static class ExternalSystemCommands
var authTypeOption = new Option<string>("--auth-type") { Description = "Auth type", Required = true };
var authConfigOption = new Option<string?>("--auth-config") { Description = "Auth configuration JSON" };
var timeoutOption = new Option<int?>("--timeout-seconds") { Description = "Per-system HTTP timeout in seconds (0 = use gateway default); omit to preserve" };
var maxRetriesOption = new Option<int?>("--max-retries") { Description = "Store-and-forward retry budget for cached calls (0 = use engine default); omit to preserve" };
var retryDelayOption = new Option<int?>("--retry-delay-seconds") { Description = "Fixed interval between store-and-forward retries, in seconds (0 = use engine default); omit to preserve" };
retryDelayOption.Aliases.Add("--retry-delay");
var cmd = new Command("update") { Description = "Update an external system" };
cmd.Add(idOption);
@@ -58,6 +61,8 @@ public static class ExternalSystemCommands
cmd.Add(authTypeOption);
cmd.Add(authConfigOption);
cmd.Add(timeoutOption);
cmd.Add(maxRetriesOption);
cmd.Add(retryDelayOption);
cmd.SetAction(async (ParseResult result) =>
{
var id = result.GetValue(idOption);
@@ -66,9 +71,13 @@ public static class ExternalSystemCommands
var authType = result.GetValue(authTypeOption)!;
var authConfig = result.GetValue(authConfigOption);
var timeoutSeconds = result.GetValue(timeoutOption);
var maxRetries = result.GetValue(maxRetriesOption);
var retryDelaySeconds = result.GetValue(retryDelayOption);
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
new UpdateExternalSystemCommand(id, name, endpointUrl, authType, authConfig, timeoutSeconds));
new UpdateExternalSystemCommand(id, name, endpointUrl, authType, authConfig, timeoutSeconds,
maxRetries,
retryDelaySeconds is { } rd ? TimeSpan.FromSeconds(rd) : null));
});
return cmd;
}
@@ -91,6 +100,9 @@ public static class ExternalSystemCommands
var authTypeOption = new Option<string>("--auth-type") { Description = "Auth type (ApiKey, BasicAuth)", Required = true };
var authConfigOption = new Option<string?>("--auth-config") { Description = "Auth configuration JSON" };
var timeoutOption = new Option<int?>("--timeout-seconds") { Description = "Per-system HTTP timeout in seconds (0 = use gateway default)" };
var maxRetriesOption = new Option<int?>("--max-retries") { Description = "Store-and-forward retry budget for cached calls (0 = use engine default)" };
var retryDelayOption = new Option<int?>("--retry-delay-seconds") { Description = "Fixed interval between store-and-forward retries, in seconds (0 = use engine default)" };
retryDelayOption.Aliases.Add("--retry-delay");
var cmd = new Command("create") { Description = "Create an external system" };
cmd.Add(nameOption);
@@ -98,6 +110,8 @@ public static class ExternalSystemCommands
cmd.Add(authTypeOption);
cmd.Add(authConfigOption);
cmd.Add(timeoutOption);
cmd.Add(maxRetriesOption);
cmd.Add(retryDelayOption);
cmd.SetAction(async (ParseResult result) =>
{
var name = result.GetValue(nameOption)!;
@@ -105,9 +119,13 @@ public static class ExternalSystemCommands
var authType = result.GetValue(authTypeOption)!;
var authConfig = result.GetValue(authConfigOption);
var timeoutSeconds = result.GetValue(timeoutOption);
var maxRetries = result.GetValue(maxRetriesOption);
var retryDelaySeconds = result.GetValue(retryDelayOption);
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
new CreateExternalSystemCommand(name, endpointUrl, authType, authConfig, timeoutSeconds));
new CreateExternalSystemCommand(name, endpointUrl, authType, authConfig, timeoutSeconds,
maxRetries,
retryDelaySeconds is { } rd ? TimeSpan.FromSeconds(rd) : null));
});
return cmd;
}
+17 -3
View File
@@ -1206,7 +1206,7 @@ scadabridge --url <url> external-system list
Register an external HTTP system that scripts can call.
```sh
scadabridge --url <url> external-system create --name <string> --endpoint-url <url> --auth-type <string> [--auth-config <json>]
scadabridge --url <url> external-system create --name <string> --endpoint-url <url> --auth-type <string> [--auth-config <json>] [--timeout-seconds <int>] [--max-retries <int>] [--retry-delay-seconds <int>]
```
| Option | Required | Description |
@@ -1215,6 +1215,13 @@ scadabridge --url <url> external-system create --name <string> --endpoint-url <u
| `--endpoint-url` | yes | Base URL of the external system |
| `--auth-type` | yes | Authentication type: `ApiKey` or `BasicAuth` |
| `--auth-config` | no | Auth credentials as a JSON string |
| `--timeout-seconds` | no | Per-system HTTP round-trip timeout. `0` (default) = use the gateway's `DefaultHttpTimeout` |
| `--max-retries` | no | Store-and-forward retry budget for this system's `CachedCall`s. `0` (default) = use the engine's `DefaultMaxRetries` (50) |
| `--retry-delay-seconds` | no | Fixed interval between store-and-forward retries. `0` (default) = use the engine's `DefaultRetryInterval` (30 s). Alias: `--retry-delay` |
The retry options are deployed to sites with the rest of the definition, so they take
effect on the next deployment — they govern how long a transient outage of this system is
buffered before a cached call parks (`--max-retries` x `--retry-delay-seconds`).
#### `external-system update`
@@ -1222,7 +1229,7 @@ Update an external system definition. An update **replaces** the whole entity
required field below must be supplied, even if unchanged.
```sh
scadabridge --url <url> external-system update --id <int> --name <string> --endpoint-url <url> --auth-type <string> [--auth-config <json>]
scadabridge --url <url> external-system update --id <int> --name <string> --endpoint-url <url> --auth-type <string> [--auth-config <json>] [--timeout-seconds <int>] [--max-retries <int>] [--retry-delay-seconds <int>]
```
| Option | Required | Description |
@@ -1231,7 +1238,14 @@ scadabridge --url <url> external-system update --id <int> --name <string> --endp
| `--name` | yes | Display name |
| `--endpoint-url` | yes | Base URL |
| `--auth-type` | yes | Authentication type |
| `--auth-config` | no | Updated auth credentials as a JSON string |
| `--auth-config` | no | Updated auth credentials as a JSON string; omit to preserve the stored value |
| `--timeout-seconds` | no | Per-system HTTP round-trip timeout; omit to preserve |
| `--max-retries` | no | Store-and-forward retry budget for this system's `CachedCall`s; omit to preserve |
| `--retry-delay-seconds` | no | Fixed interval between store-and-forward retries; omit to preserve. Alias: `--retry-delay` |
Note the asymmetry with the required fields above: the four optional settings follow an
**omit-to-preserve** rule, so a rename does not silently reset a system's auth config,
timeout, or retry policy.
#### `external-system delete`
@@ -1,9 +1,41 @@
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
/// <summary>
/// A single external HTTP system, flattened for deployment to a site.
/// </summary>
/// <param name="Name">Unique system name — the key scripts call it by.</param>
/// <param name="EndpointUrl">Base URL of the system's HTTP endpoint.</param>
/// <param name="AuthType">Authentication type identifier (e.g. <c>ApiKey</c>, <c>BasicAuth</c>).</param>
/// <param name="AuthConfiguration">JSON-serialized authentication configuration for <paramref name="AuthType"/>.</param>
/// <param name="MethodDefinitionsJson">JSON-serialized method list, or <c>null</c> when the system defines none.</param>
/// <param name="TimeoutSeconds">
/// Per-system HTTP round-trip timeout in seconds. <c>0</c> = unset — the gateway's configured
/// <c>DefaultHttpTimeout</c> applies.
/// </param>
/// <param name="MaxRetries">
/// Store-and-Forward retry budget for this system's cached calls. <c>0</c> = unset — the engine's
/// <c>DefaultMaxRetries</c> applies.
/// <para>
/// <b>0 means "unset", not "no limit", at THIS layer</b> — the inverse of
/// <c>StoreAndForwardMessage.MaxRetries</c>, where a stored 0 means unbounded retry. The two
/// meanings are reconciled in <c>ExternalSystemClient.CachedCallAsync</c>, which passes a 0
/// through as <c>null</c> so the bounded engine default applies rather than an unbounded retry
/// loop. Keeping "unset" as 0 here (rather than a nullable) matches
/// <see cref="DatabaseConnectionArtifact"/>, the non-nullable
/// <c>ExternalSystemDefinition.MaxRetries</c> entity property this is projected from, and the
/// proto3 scalar it is carried on.
/// </para>
/// </param>
/// <param name="RetryDelay">
/// Fixed interval between Store-and-Forward retry sweeps for this system's cached calls.
/// <see cref="TimeSpan.Zero"/> = unset — the engine's <c>DefaultRetryInterval</c> applies.
/// </param>
public record ExternalSystemArtifact(
string Name,
string EndpointUrl,
string AuthType,
string? AuthConfiguration,
string? MethodDefinitionsJson,
int TimeoutSeconds = 0);
int TimeoutSeconds = 0,
int MaxRetries = 0,
TimeSpan RetryDelay = default);
@@ -2,8 +2,23 @@ namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
public record ListExternalSystemsCommand;
public record GetExternalSystemCommand(int ExternalSystemId);
public record CreateExternalSystemCommand(string Name, string EndpointUrl, string AuthType, string? AuthConfiguration, int? TimeoutSeconds = null);
public record UpdateExternalSystemCommand(int ExternalSystemId, string Name, string EndpointUrl, string AuthType, string? AuthConfiguration, int? TimeoutSeconds = null);
/// <summary>
/// Creates an external HTTP system.
/// </summary>
/// <remarks>
/// <paramref name="MaxRetries"/> and <paramref name="RetryDelay"/> are the Store-and-Forward
/// retry policy applied to this system's <c>ExternalSystem.CachedCall</c> buffering. Both are
/// nullable on the wire so "omit" is distinguishable from "set to 0"; a null (or an explicit 0)
/// stores the entity's unset value, which the site gateway translates into the engine's
/// <c>DefaultMaxRetries</c>/<c>DefaultRetryInterval</c>.
/// </remarks>
public record CreateExternalSystemCommand(string Name, string EndpointUrl, string AuthType, string? AuthConfiguration, int? TimeoutSeconds = null, int? MaxRetries = null, TimeSpan? RetryDelay = null);
/// <summary>
/// Updates an external HTTP system. Nullable members follow the "omit to preserve" convention —
/// a null leaves the stored value untouched.
/// </summary>
public record UpdateExternalSystemCommand(int ExternalSystemId, string Name, string EndpointUrl, string AuthType, string? AuthConfiguration, int? TimeoutSeconds = null, int? MaxRetries = null, TimeSpan? RetryDelay = null);
public record DeleteExternalSystemCommand(int ExternalSystemId);
// External System Methods
@@ -506,7 +506,9 @@ public static class SiteCommandDtoMapper
AuthType = artifact.AuthType,
AuthConfiguration = OrEmpty(artifact.AuthConfiguration),
MethodDefinitionsJson = OrEmpty(artifact.MethodDefinitionsJson),
TimeoutSeconds = artifact.TimeoutSeconds
TimeoutSeconds = artifact.TimeoutSeconds,
MaxRetries = artifact.MaxRetries,
RetryDelay = Duration.FromTimeSpan(artifact.RetryDelay)
};
}
@@ -523,7 +525,9 @@ public static class SiteCommandDtoMapper
dto.AuthType,
NullIfEmpty(dto.AuthConfiguration),
NullIfEmpty(dto.MethodDefinitionsJson),
dto.TimeoutSeconds);
dto.TimeoutSeconds,
dto.MaxRetries,
dto.RetryDelay?.ToTimeSpan() ?? TimeSpan.Zero);
}
/// <summary>Projects a <see cref="DatabaseConnectionArtifact"/> onto the wire.</summary>
@@ -224,6 +224,11 @@ message ExternalSystemArtifactDto {
string auth_configuration = 4; // empty string represents null
string method_definitions_json = 5; // empty string represents null
int32 timeout_seconds = 6;
// Store-and-Forward retry policy for this system's cached calls. Additive (fields 7-8);
// a pre-existing site decodes them as 0/absent, which is exactly the "unset" value that
// makes the engine defaults apply — i.e. the old behaviour.
int32 max_retries = 7;
google.protobuf.Duration retry_delay = 8;
}
message DatabaseConnectionArtifactDto {
File diff suppressed because it is too large Load Diff
@@ -192,8 +192,13 @@ public class ArtifactDeploymentService
m.ReturnDefinition
}))
: null;
// MaxRetries/RetryDelay ride along so the site's Store-and-Forward buffering honours
// the centrally-configured per-system retry policy. Before they were carried, a site
// silently fell back to the engine defaults (50 x 30 s) for every cached call, whatever
// central had configured — the DatabaseConnectionArtifact below always carried them.
externalSystemArtifacts.Add(new ExternalSystemArtifact(
es.Name, es.EndpointUrl, es.AuthType, es.AuthConfiguration, methodsJson, es.TimeoutSeconds));
es.Name, es.EndpointUrl, es.AuthType, es.AuthConfiguration, methodsJson, es.TimeoutSeconds,
es.MaxRetries, es.RetryDelay));
}
// Map database connections
@@ -113,7 +113,7 @@ public static class SiteLocalDbLegacyMigrator
new("external_systems", "name",
[
"name", "endpoint_url", "auth_type", "auth_configuration", "method_definitions",
"updated_at", "timeout_seconds",
"updated_at", "timeout_seconds", "max_retries", "retry_delay_ms",
]),
new("database_connections", "name",
[
@@ -2021,7 +2021,11 @@ public class ManagementActor : ReceiveActor
var def = new ExternalSystemDefinition(cmd.Name, cmd.EndpointUrl, cmd.AuthType)
{
AuthConfiguration = cmd.AuthConfiguration,
TimeoutSeconds = cmd.TimeoutSeconds ?? 0
TimeoutSeconds = cmd.TimeoutSeconds ?? 0,
// 0 / TimeSpan.Zero is the entity's "unset" value — the site gateway maps it to the
// Store-and-Forward engine defaults rather than to a literal zero budget.
MaxRetries = cmd.MaxRetries ?? 0,
RetryDelay = cmd.RetryDelay ?? TimeSpan.Zero
};
await repo.AddExternalSystemAsync(def);
await repo.SaveChangesAsync();
@@ -2042,6 +2046,9 @@ public class ManagementActor : ReceiveActor
// secret intact (mirrors the SMTP/SMS credential preserve-if-null rule).
if (cmd.AuthConfiguration is not null) def.AuthConfiguration = cmd.AuthConfiguration;
if (cmd.TimeoutSeconds is { } ts) def.TimeoutSeconds = ts;
// Same preserve-if-null rule for the Store-and-Forward retry policy.
if (cmd.MaxRetries is { } mr) def.MaxRetries = mr;
if (cmd.RetryDelay is { } rd) def.RetryDelay = rd;
await repo.UpdateExternalSystemAsync(def);
await repo.SaveChangesAsync();
var publicShape = ExternalSystemPublicShape(def);
@@ -2155,7 +2155,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
foreach (var es in command.ExternalSystems)
{
await _storage.StoreExternalSystemAsync(es.Name, es.EndpointUrl,
es.AuthType, es.AuthConfiguration, es.MethodDefinitionsJson, es.TimeoutSeconds);
es.AuthType, es.AuthConfiguration, es.MethodDefinitionsJson, es.TimeoutSeconds,
es.MaxRetries, es.RetryDelay);
}
await _storage.DeleteExternalSystemsExceptAsync(
@@ -133,6 +133,28 @@ public static class SiteStorageSchema
// Per-external-system call timeout (ExternalSystemGateway Timeout) — carried through the
// artifact pipeline so site-side calls honor it; 0 = use the site default.
AddColumnIfMissing(connection, "external_systems", "timeout_seconds", "INTEGER NOT NULL DEFAULT 0");
// Store-and-Forward retry policy, deployed per external system (deferred-work row 27).
//
// 0 = unset on BOTH columns, which is what makes this additive change behaviourally
// invisible until an operator actually configures a policy: an existing row backfilled
// with the DEFAULT reads back as 0, and ExternalSystemClient.CachedCallAsync maps a 0
// to null so the engine's DefaultMaxRetries/DefaultRetryInterval still apply. (Contrast
// database_connections, whose equivalent columns default to 3/1000 — that table has
// never had an "unset means engine default" contract.)
//
// These land on a table in SiteLocalDbSetup.ReplicatedTables. That is safe and needs no
// trigger migration: RegisterReplicated re-reads PRAGMA table_info on EVERY boot and its
// install script is DROP-then-CREATE, so the capture triggers are rebuilt over the new
// column set automatically — and Apply() runs before RegisterReplicated, so the rebuild
// sees these columns on the same boot that adds them. What DOES change is the table's
// schema digest (ReplicatedTable.ComputeDigest hashes every column name + type), which
// the sync handshake compares fail-closed; a pair running mismatched builds refuses to
// sync loudly (LocalDbSchemaMismatchException) rather than corrupting anything. Both
// nodes must therefore be upgraded together — already the standing rule for a site pair
// (docs/deployment/topology-guide.md), so this introduces no new operational constraint.
AddColumnIfMissing(connection, "external_systems", "max_retries", "INTEGER NOT NULL DEFAULT 0");
AddColumnIfMissing(connection, "external_systems", "retry_delay_ms", "INTEGER NOT NULL DEFAULT 0");
}
/// <summary>
@@ -647,23 +647,33 @@ public class SiteStorageService
/// <param name="authConfig">Authentication configuration JSON, if applicable.</param>
/// <param name="methodDefs">JSON representation of available method definitions, if any.</param>
/// <param name="timeoutSeconds">Per-system call timeout in seconds (0 = use the site default).</param>
/// <param name="maxRetries">
/// Store-and-Forward retry budget for this system's cached calls (0 = unset — the engine's
/// <c>DefaultMaxRetries</c> applies).
/// </param>
/// <param name="retryDelay">
/// Fixed interval between Store-and-Forward retry sweeps for this system's cached calls
/// (<see cref="TimeSpan.Zero"/> = unset — the engine's <c>DefaultRetryInterval</c> applies).
/// </param>
/// <returns>A task that completes when the external system definition has been stored or updated.</returns>
public async Task StoreExternalSystemAsync(
string name, string endpointUrl, string authType, string? authConfig, string? methodDefs,
int timeoutSeconds = 0)
int timeoutSeconds = 0, int maxRetries = 0, TimeSpan retryDelay = default)
{
await using var connection = OpenConnection();
await using var command = connection.CreateCommand();
command.CommandText = @"
INSERT INTO external_systems (name, endpoint_url, auth_type, auth_configuration, method_definitions, timeout_seconds, updated_at)
VALUES (@name, @url, @authType, @authConfig, @methodDefs, @timeoutSeconds, @updatedAt)
INSERT INTO external_systems (name, endpoint_url, auth_type, auth_configuration, method_definitions, timeout_seconds, max_retries, retry_delay_ms, updated_at)
VALUES (@name, @url, @authType, @authConfig, @methodDefs, @timeoutSeconds, @maxRetries, @retryDelayMs, @updatedAt)
ON CONFLICT(name) DO UPDATE SET
endpoint_url = excluded.endpoint_url,
auth_type = excluded.auth_type,
auth_configuration = excluded.auth_configuration,
method_definitions = excluded.method_definitions,
timeout_seconds = excluded.timeout_seconds,
max_retries = excluded.max_retries,
retry_delay_ms = excluded.retry_delay_ms,
updated_at = excluded.updated_at";
command.Parameters.AddWithValue("@name", name);
@@ -672,6 +682,8 @@ public class SiteStorageService
command.Parameters.AddWithValue("@authConfig", (object?)authConfig ?? DBNull.Value);
command.Parameters.AddWithValue("@methodDefs", (object?)methodDefs ?? DBNull.Value);
command.Parameters.AddWithValue("@timeoutSeconds", timeoutSeconds);
command.Parameters.AddWithValue("@maxRetries", maxRetries);
command.Parameters.AddWithValue("@retryDelayMs", (long)retryDelay.TotalMilliseconds);
command.Parameters.AddWithValue("@updatedAt", DateTimeOffset.UtcNow.ToString("O"));
await command.ExecuteNonQueryAsync();
@@ -250,7 +250,13 @@ public class SiteExternalSystemRepository : IExternalSystemRepository
{
Id = GenerateSyntheticId(name),
AuthConfiguration = reader.IsDBNull(3) ? null : reader.GetString(3),
TimeoutSeconds = reader.IsDBNull(4) ? 0 : reader.GetInt32(4)
TimeoutSeconds = reader.IsDBNull(4) ? 0 : reader.GetInt32(4),
// Deployed Store-and-Forward retry policy (deferred-work row 27). Ordinals 6/7 are
// deliberately AFTER method_definitions (5) so the pre-existing indexes are untouched.
// 0 stays 0: it is the "unset" value ExternalSystemClient.CachedCallAsync converts to
// null so the engine defaults apply.
MaxRetries = reader.IsDBNull(6) ? 0 : reader.GetInt32(6),
RetryDelay = reader.IsDBNull(7) ? TimeSpan.Zero : TimeSpan.FromMilliseconds(reader.GetInt64(7))
};
}
@@ -271,7 +277,8 @@ public class SiteExternalSystemRepository : IExternalSystemRepository
await using var command = connection.CreateCommand();
command.CommandText = @"
SELECT name, endpoint_url, auth_type, auth_configuration, timeout_seconds, method_definitions
SELECT name, endpoint_url, auth_type, auth_configuration, timeout_seconds, method_definitions,
max_retries, retry_delay_ms
FROM external_systems";
var byName = new Dictionary<string, ExternalSystemDefinition>(StringComparer.Ordinal);
@@ -165,6 +165,11 @@ public sealed class ArtifactDiff
var changes = new List<FieldChange>();
AddIfDifferent(changes, "BaseUrl", existing.EndpointUrl, incoming.BaseUrl);
AddIfDifferent(changes, "AuthType", existing.AuthType, incoming.AuthType);
// The bundle has always CARRIED the retry policy and the importer has always APPLIED it,
// but the diff omitted it — so an import that silently changed a system's retry budget
// previewed as "Identical". Mirrors CompareDatabaseConnection below.
AddIfDifferent(changes, "MaxRetries", existing.MaxRetries, incoming.MaxRetries);
AddIfDifferent(changes, "RetryDelay", existing.RetryDelay.ToString(), incoming.RetryDelay.ToString());
// Secrets: presence-only comparison (we never echo the value in the diff).
var existingHasSecret = !string.IsNullOrEmpty(existing.AuthConfiguration);