diff --git a/docs/requirements/Component-ExternalSystemGateway.md b/docs/requirements/Component-ExternalSystemGateway.md index 2e100873..27cec1ab 100644 --- a/docs/requirements/Component-ExternalSystemGateway.md +++ b/docs/requirements/Component-ExternalSystemGateway.md @@ -120,6 +120,8 @@ Scripts choose between two call modes per invocation, mirroring the dual-mode da - Database connections use **standard ADO.NET connection pooling** per named connection. No custom pool management. - Pool behavior (max pool size, connection lifetime, etc.) can be tuned via connection string parameters in the database connection definition if needed. +- **Definition cache (central only).** `ExternalDefinitionLookupCache` holds resolved external-system, method and database-connection definitions for `ExternalSystemGateway:DefinitionCacheTtl` (default 0 = off in the options; the central host sets 10 s because each lookup there is a SQL Server round-trip). Only found definitions are cached, so a new definition is visible immediately; an edit/delete within one TTL. Sites keep it off — `SiteExternalSystemRepository` already serves external systems from the deploy-invalidated `ExternalSystemDefinitionCache` snapshot. +- GET/DELETE parameters are rendered into the query string with scalars via `ToString()` and collections/objects JSON-serialised (never a CLR type name). - Synchronous failures on `Database.Connection()` (e.g., unreachable server) return an error to the calling script, consistent with external system permanent failure handling. ## Dependencies diff --git a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/DatabaseGateway.cs b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/DatabaseGateway.cs index 24557fe2..03a0597c 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/DatabaseGateway.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/DatabaseGateway.cs @@ -21,6 +21,7 @@ public class DatabaseGateway : IDatabaseGateway, ISandboxDatabaseConnectionSourc private readonly IExternalSystemRepository _repository; private readonly StoreAndForwardService? _storeAndForward; private readonly ILogger _logger; + private readonly ExternalDefinitionLookupCache? _definitionCache; /// /// Initializes a new instance of . @@ -28,11 +29,14 @@ public class DatabaseGateway : IDatabaseGateway, ISandboxDatabaseConnectionSourc /// Repository for resolving database connection definitions. /// Logger for diagnostics. /// Optional store-and-forward service for cached writes; null disables buffering. + /// Optional short-TTL definition cache; null (or a zero TTL) resolves every call from the repository. public DatabaseGateway( IExternalSystemRepository repository, ILogger logger, - StoreAndForwardService? storeAndForward = null) + StoreAndForwardService? storeAndForward = null, + ExternalDefinitionLookupCache? definitionCache = null) { + _definitionCache = definitionCache; _repository = repository; _logger = logger; _storeAndForward = storeAndForward; @@ -447,6 +451,12 @@ public class DatabaseGateway : IDatabaseGateway, ISandboxDatabaseConnectionSourc // fetch-all-then-filter — connection definitions are resolved on every // cached write / connection request, so the repository performs an indexed // query rather than loading every connection into memory. + if (_definitionCache is { Enabled: true }) + { + return await _definitionCache.GetConnectionAsync(connectionName, + () => _repository.GetDatabaseConnectionByNameAsync(connectionName, cancellationToken)); + } + return await _repository.GetDatabaseConnectionByNameAsync(connectionName, cancellationToken); } } diff --git a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalDefinitionLookupCache.cs b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalDefinitionLookupCache.cs new file mode 100644 index 00000000..8b8467cd --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalDefinitionLookupCache.cs @@ -0,0 +1,98 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.ScadaBridge.Commons.Entities.ExternalSystems; + +namespace ZB.MOM.WW.ScadaBridge.ExternalSystemGateway; + +/// +/// Short-TTL, in-memory cache of the external-system / method / database-connection +/// definitions the gateway resolves on every ExternalSystem.Call and +/// Database.* invocation (review 2026-10-21 05/P2). +/// +/// Enabled only when is +/// positive — the central host turns it on (each call there is otherwise a SQL Server +/// round-trip inside an inbound request). Site nodes leave it off: their +/// SiteExternalSystemRepository already serves external systems from the shared, +/// deploy-invalidated ExternalSystemDefinitionCache snapshot, and layering a TTL on +/// top would only delay the effect of an artifact deploy. Only found definitions are cached +/// (a miss always re-queries), so a newly created system is visible immediately; an edited or +/// deleted one is visible within one TTL. +/// +/// +public sealed class ExternalDefinitionLookupCache +{ + private sealed record Entry(object? Value, DateTimeOffset ExpiresAtUtc); + + private const int PruneThreshold = 512; + + private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); + private readonly TimeSpan _ttl; + private readonly TimeProvider _timeProvider; + + /// Initializes the cache from the gateway options. + /// Gateway options; sets the TTL. + /// Optional time source; defaults to . + public ExternalDefinitionLookupCache(IOptions options, TimeProvider? timeProvider = null) + : this(options.Value.DefinitionCacheTtl, timeProvider) + { + } + + /// Initializes the cache with an explicit TTL. + /// Entry lifetime; zero or negative disables caching. + /// Optional time source; defaults to . + public ExternalDefinitionLookupCache(TimeSpan ttl, TimeProvider? timeProvider = null) + { + _ttl = ttl; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// Whether caching is active. + public bool Enabled => _ttl > TimeSpan.Zero; + + /// Resolves an external system by name, via the cache. + /// The system name. + /// Loads the definition on a miss. + /// The definition, or null when it does not exist. + public Task GetSystemAsync(string name, Func> fetch) + => GetAsync("s:" + name, fetch); + + /// Resolves a method of an external system by name, via the cache. + /// The owning system's id. + /// The method name. + /// Loads the method on a miss. + /// The method, or null when it does not exist. + public Task GetMethodAsync(int systemId, string name, Func> fetch) + => GetAsync("m:" + systemId + ":" + name, fetch); + + /// Resolves a database connection definition by name, via the cache. + /// The connection name. + /// Loads the definition on a miss. + /// The definition, or null when it does not exist. + public Task GetConnectionAsync(string name, Func> fetch) + => GetAsync("c:" + name, fetch); + + /// Drops every cached definition. + public void InvalidateAll() => _entries.Clear(); + + private async Task GetAsync(string key, Func> fetch) where T : class + { + if (!Enabled) + return await fetch().ConfigureAwait(false); + + var now = _timeProvider.GetUtcNow(); + if (_entries.TryGetValue(key, out var hit) && hit.ExpiresAtUtc > now && hit.Value is T cached) + return cached; + + var value = await fetch().ConfigureAwait(false); + if (value is not null) + { + if (_entries.Count >= PruneThreshold) + { + foreach (var kv in _entries) + if (kv.Value.ExpiresAtUtc <= now) _entries.TryRemove(kv); + } + _entries[key] = new Entry(value, now + _ttl); + } + return value; + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemClient.cs b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemClient.cs index 324654a8..357b628a 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemClient.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemClient.cs @@ -26,6 +26,7 @@ public class ExternalSystemClient : IExternalSystemClient private readonly StoreAndForwardService? _storeAndForward; private readonly ILogger _logger; private readonly ExternalSystemGatewayOptions _options; + private readonly ExternalDefinitionLookupCache? _definitionCache; /// /// Shared options for the per-call AuthConfiguration JSON parse in @@ -44,13 +45,16 @@ public class ExternalSystemClient : IExternalSystemClient /// Logger instance. /// Store-and-forward service for buffering transient failures, or null if not available. /// Configuration options, or null for defaults. + /// Optional short-TTL definition cache; null (or a zero TTL) resolves every call from the repository. public ExternalSystemClient( IHttpClientFactory httpClientFactory, IExternalSystemRepository repository, ILogger logger, StoreAndForwardService? storeAndForward = null, - IOptions? options = null) + IOptions? options = null, + ExternalDefinitionLookupCache? definitionCache = null) { + _definitionCache = definitionCache; _httpClientFactory = httpClientFactory; _repository = repository; _logger = logger; @@ -815,11 +819,17 @@ public class ExternalSystemClient : IExternalSystemClient // fetch-all-then-filter — definitions are resolved on every hot-path call // (a script's ExternalSystem.Call()), so the repository performs an indexed // query rather than loading every system / every method into memory. - var system = await _repository.GetExternalSystemByNameAsync(systemName, cancellationToken); + var system = _definitionCache is { Enabled: true } + ? await _definitionCache.GetSystemAsync(systemName, + () => _repository.GetExternalSystemByNameAsync(systemName, cancellationToken)) + : await _repository.GetExternalSystemByNameAsync(systemName, cancellationToken); if (system == null) return (null, null); - var method = await _repository.GetMethodByNameAsync(system.Id, methodName, cancellationToken); + var method = _definitionCache is { Enabled: true } + ? await _definitionCache.GetMethodAsync(system.Id, methodName, + () => _repository.GetMethodByNameAsync(system.Id, methodName, cancellationToken)) + : await _repository.GetMethodByNameAsync(system.Id, methodName, cancellationToken); return (system, method); } diff --git a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemGatewayOptions.cs b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemGatewayOptions.cs index 7e0e4c75..eb7edd42 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemGatewayOptions.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemGatewayOptions.cs @@ -19,4 +19,13 @@ public class ExternalSystemGatewayOptions /// Default 10 MiB. /// public long MaxResponseBodyBytes { get; set; } = 10 * 1024 * 1024; + + /// + /// How long a resolved external-system / method / database-connection definition is + /// cached in memory before the repository is consulted again. + /// (the default) disables the cache. The central host defaults it on (10 s) because each + /// lookup there is a SQL Server round-trip; site nodes keep it off since their repository + /// is already a deploy-invalidated snapshot (review 2026-10-21 05/P2). + /// + public TimeSpan DefinitionCacheTtl { get; set; } = TimeSpan.Zero; } diff --git a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemGatewayOptionsValidator.cs b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemGatewayOptionsValidator.cs index c3d7a25d..ab715d03 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemGatewayOptionsValidator.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ExternalSystemGatewayOptionsValidator.cs @@ -29,5 +29,9 @@ public sealed class ExternalSystemGatewayOptionsValidator : OptionsValidatorBase builder.RequireThat(options.MaxResponseBodyBytes > 0, $"ScadaBridge:ExternalSystemGateway:MaxResponseBodyBytes must be greater than zero " + $"(was {options.MaxResponseBodyBytes}); it caps the outbound response body buffered into memory."); + + builder.RequireThat(options.DefinitionCacheTtl >= TimeSpan.Zero, + $"ScadaBridge:ExternalSystemGateway:DefinitionCacheTtl must not be negative " + + $"(was {options.DefinitionCacheTtl}); use 00:00:00 to disable the definition cache."); } } diff --git a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ServiceCollectionExtensions.cs b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ServiceCollectionExtensions.cs index ccf78534..e63f04e7 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/ServiceCollectionExtensions.cs @@ -43,6 +43,7 @@ public static class ServiceCollectionExtensions new GatewayHttpClientConfigurator( sp.GetRequiredService>())); + services.TryAddSingleton(); services.AddScoped(); services.AddScoped(sp => sp.GetRequiredService()); services.AddScoped(); diff --git a/tests/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway.Tests/ExternalDefinitionLookupCacheTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway.Tests/ExternalDefinitionLookupCacheTests.cs new file mode 100644 index 00000000..9da8c53c --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway.Tests/ExternalDefinitionLookupCacheTests.cs @@ -0,0 +1,55 @@ +using ZB.MOM.WW.ScadaBridge.Commons.Entities.ExternalSystems; + +namespace ZB.MOM.WW.ScadaBridge.ExternalSystemGateway.Tests; + +/// Tests for (review 2026-10-21 05/P2). +public class ExternalDefinitionLookupCacheTests +{ + private static ExternalSystemDefinition Sys() => new("S", "https://x", "none") { Id = 1 }; + + /// Verifies hits within the TTL skip the repository and expiry refetches. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Hit_WithinTtl_SkipsFetch_AndExpiryRefetches() + { + var time = new FakeTimeProvider(DateTimeOffset.UtcNow); + var cache = new ExternalDefinitionLookupCache(TimeSpan.FromSeconds(5), time); + var fetches = 0; + Task Fetch() { fetches++; return Task.FromResult(Sys()); } + + await cache.GetSystemAsync("S", Fetch); + await cache.GetSystemAsync("S", Fetch); + Assert.Equal(1, fetches); + + time.Advance(TimeSpan.FromSeconds(6)); + await cache.GetSystemAsync("S", Fetch); + Assert.Equal(2, fetches); + } + + /// Verifies misses are not cached and a zero TTL disables caching. + /// A task that represents the asynchronous operation. + [Fact] + public async Task NotFound_IsNotCached_AndZeroTtlDisables() + { + var cache = new ExternalDefinitionLookupCache(TimeSpan.FromSeconds(5)); + var fetches = 0; + Task Miss() { fetches++; return Task.FromResult(null); } + await cache.GetSystemAsync("S", Miss); + await cache.GetSystemAsync("S", Miss); + Assert.Equal(2, fetches); + + var off = new ExternalDefinitionLookupCache(TimeSpan.Zero); + fetches = 0; + Task Hit() { fetches++; return Task.FromResult(Sys()); } + await off.GetSystemAsync("S", Hit); + await off.GetSystemAsync("S", Hit); + Assert.Equal(2, fetches); + } + + private sealed class FakeTimeProvider(DateTimeOffset start) : TimeProvider + { + private DateTimeOffset _now = start; + public override DateTimeOffset GetUtcNow() => _now; + public void Advance(TimeSpan by) => _now += by; + } +}