fix(external-system-gateway): resolve ExternalSystemGateway-004..010 — honour retry settings, dispose HTTP messages, fix URL building, truncate error bodies, fix connection leak

This commit is contained in:
Joseph Doherty
2026-05-16 21:11:24 -04:00
parent 8c67ffad2a
commit 2502e4d10a
5 changed files with 615 additions and 52 deletions

View File

@@ -45,11 +45,29 @@ public class DatabaseGateway : IDatabaseGateway
throw new InvalidOperationException($"Database connection '{connectionName}' not found");
}
var connection = new SqlConnection(definition.ConnectionString);
await connection.OpenAsync(cancellationToken);
var connection = CreateConnection(definition.ConnectionString);
try
{
await connection.OpenAsync(cancellationToken);
}
catch
{
// OpenAsync failed (unreachable server, bad credentials, cancellation) —
// dispose the just-created connection before the exception propagates so
// it is not leaked (ExternalSystemGateway-010).
await connection.DisposeAsync();
throw;
}
return connection;
}
/// <summary>
/// Creates the underlying ADO.NET connection for a connection string. Virtual so
/// tests can substitute a connection whose <c>OpenAsync</c> fails.
/// </summary>
internal virtual DbConnection CreateConnection(string connectionString) =>
new SqlConnection(connectionString);
/// <summary>
/// Submits a SQL write to the store-and-forward engine for reliable delivery.
/// </summary>
@@ -78,12 +96,15 @@ public class DatabaseGateway : IDatabaseGateway
Parameters = parameters
});
// The per-connection retry settings are passed through verbatim — a
// configured MaxRetries of 0 means "never retry" and must NOT be
// collapsed to the S&F default (ExternalSystemGateway-004).
await _storeAndForward.EnqueueAsync(
StoreAndForwardCategory.CachedDbWrite,
connectionName,
payload,
originInstanceName,
definition.MaxRetries > 0 ? definition.MaxRetries : null,
definition.MaxRetries,
definition.RetryDelay > TimeSpan.Zero ? definition.RetryDelay : null);
}

View File

@@ -113,12 +113,16 @@ public class ExternalSystemClient : IExternalSystemClient
// attemptImmediateDelivery: false — this method already made the HTTP
// attempt above; letting EnqueueAsync re-invoke the handler would
// dispatch the same request a second time.
//
// The per-system retry settings are passed through verbatim — a
// configured MaxRetries of 0 means "never retry" and must NOT be
// collapsed to the S&F default (ExternalSystemGateway-004).
await _storeAndForward.EnqueueAsync(
StoreAndForwardCategory.ExternalSystem,
systemName,
payload,
originInstanceName,
system.MaxRetries > 0 ? system.MaxRetries : null,
system.MaxRetries,
system.RetryDelay > TimeSpan.Zero ? system.RetryDelay : null,
attemptImmediateDelivery: false);
@@ -183,7 +187,11 @@ public class ExternalSystemClient : IExternalSystemClient
var client = _httpClientFactory.CreateClient($"ExternalSystem_{system.Name}");
var url = BuildUrl(system.EndpointUrl, method.Path, parameters, method.HttpMethod);
var request = new HttpRequestMessage(new HttpMethod(method.HttpMethod), url);
// The request and response own IDisposable resources (StringContent, the
// response content stream). Dispose both, including on the exception paths
// (ExternalSystemGateway-005).
using var request = new HttpRequestMessage(new HttpMethod(method.HttpMethod), url);
// Apply authentication
ApplyAuth(request, system);
@@ -232,44 +240,75 @@ public class ExternalSystemClient : IExternalSystemClient
throw ErrorClassifier.AsTransient($"Connection error to {system.Name}: {ex.Message}", ex);
}
// The timeout also covers reading the response body (the design's
// "round-trip" guarantee), so the linked token is used for the read too.
string body;
try
using (response)
{
body = await response.Content.ReadAsStringAsync(linkedCts.Token);
// The timeout also covers reading the response body (the design's
// "round-trip" guarantee), so the linked token is used for the read too.
string body;
try
{
body = await response.Content.ReadAsStringAsync(linkedCts.Token);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (OperationCanceledException ex) when (timeoutCts.IsCancellationRequested)
{
throw ErrorClassifier.AsTransient(
$"Timeout reading response from {system.Name} after {_options.DefaultHttpTimeout.TotalSeconds:0.##}s", ex);
}
if (response.IsSuccessStatusCode)
{
return body;
}
// Bound the external error body before embedding it into a
// script-visible message / event-log entry — a misbehaving or hostile
// endpoint must not be able to inflate every error string
// (ExternalSystemGateway-007).
var errorBody = Truncate(body, MaxErrorBodyChars);
if (ErrorClassifier.IsTransient(response.StatusCode))
{
throw ErrorClassifier.AsTransient(
$"HTTP {(int)response.StatusCode} from {system.Name}: {errorBody}");
}
throw new PermanentExternalSystemException(
$"HTTP {(int)response.StatusCode} from {system.Name}: {errorBody}",
(int)response.StatusCode);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
}
/// <summary>
/// Upper bound (characters) on an external error response body echoed into a
/// script-visible error message — see ExternalSystemGateway-007.
/// </summary>
private const int MaxErrorBodyChars = 2048;
private static string Truncate(string value, int maxChars)
{
if (string.IsNullOrEmpty(value) || value.Length <= maxChars)
{
throw;
}
catch (OperationCanceledException ex) when (timeoutCts.IsCancellationRequested)
{
throw ErrorClassifier.AsTransient(
$"Timeout reading response from {system.Name} after {_options.DefaultHttpTimeout.TotalSeconds:0.##}s", ex);
return value;
}
if (response.IsSuccessStatusCode)
{
return body;
}
var errorBody = body;
if (ErrorClassifier.IsTransient(response.StatusCode))
{
throw ErrorClassifier.AsTransient(
$"HTTP {(int)response.StatusCode} from {system.Name}: {errorBody}");
}
throw new PermanentExternalSystemException(
$"HTTP {(int)response.StatusCode} from {system.Name}: {errorBody}",
(int)response.StatusCode);
return value.Substring(0, maxChars) + $"… [truncated, {value.Length} chars total]";
}
private static string BuildUrl(string baseUrl, string path, IReadOnlyDictionary<string, object?>? parameters, string httpMethod)
{
var url = baseUrl.TrimEnd('/') + "/" + path.TrimStart('/');
// A method that targets the base URL itself has an empty (or "/") path.
// Appending a trailing "/" in that case yields ".../api/" which some
// servers treat as a distinct resource — only append a segment when the
// method actually defines a non-empty relative path (ExternalSystemGateway-006).
var trimmedBase = baseUrl.TrimEnd('/');
var trimmedPath = path.Trim().TrimStart('/');
var url = string.IsNullOrEmpty(trimmedPath)
? trimmedBase
: trimmedBase + "/" + trimmedPath;
// For GET/DELETE, append parameters as query string
if ((httpMethod.Equals("GET", StringComparison.OrdinalIgnoreCase) ||