fix(galaxy): #875 Galaxy initialize gets a 20 s budget via per-driver-type capability defaults
Gate F (#761) against the VD03 gateway: Galaxy initialize (session open + platform seed) took 4–5 s against Tier A's 5 s Initialize timeout, so 10 of 40 restarts failed, paid a 10 s retry and left a killed gateway worker. DriverFactoryRegistry.Register takes optional capabilityDefaults per driver type (GetCapabilityDefaults); DriverResilienceOptionsParser layers them over the tier table and under the per-instance ResilienceConfig; the Host wires the registry into DriverCapabilityInvokerFactory. Galaxy registers Initialize = 20 s (no retry, breaker 2), under the actor's 30 s pessimistic bound. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -134,7 +134,12 @@ following this note. `DriverResilienceOptions.GetTierDefaults(tier)` supplies
|
||||
the real per-capability timeout / retry / breaker policies, resolved via
|
||||
`DriverFactoryRegistry.GetTier` and applied by `DriverCapabilityInvokerFactory`.
|
||||
Every driver runs Tier A because no factory passes a tier, so today that means
|
||||
one uniform policy set.
|
||||
one uniform policy set — except where a driver type registers **capability
|
||||
defaults** with its factory (`DriverFactoryRegistry.Register(...,
|
||||
capabilityDefaults:)`, read back by `GetCapabilityDefaults`). Those layer over
|
||||
the tier table and under any per-instance `ResilienceConfig`. Galaxy registers
|
||||
`Initialize` = 20 s (#875): its session open plus platform seed took 4–5 s
|
||||
against a real gateway, so Tier A's 5 s failed a quarter of restarts.
|
||||
|
||||
`IDriver` **no longer carries `GetMemoryFootprint()` or
|
||||
`FlushOptionalCachesAsync()`** (Gitea **#525**). They existed to feed
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Hosting;
|
||||
@@ -21,6 +22,8 @@ public sealed class DriverFactoryRegistry
|
||||
= new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, DriverTier> _tiers
|
||||
= new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, IReadOnlyDictionary<DriverCapability, CapabilityPolicy>> _capabilityDefaults
|
||||
= new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly object _lock = new();
|
||||
|
||||
/// <summary>
|
||||
@@ -42,8 +45,14 @@ public sealed class DriverFactoryRegistry
|
||||
/// <c>DriverInstanceBootstrapper</c> can skip in-process drivers by tier check
|
||||
/// rather than by driver-type allow-list.
|
||||
/// </param>
|
||||
/// <param name="capabilityDefaults">
|
||||
/// Optional per-driver-TYPE capability policies layered over the tier defaults (and under any
|
||||
/// per-instance <c>ResilienceConfig</c>) — for a driver whose real cost of one capability does not fit its
|
||||
/// tier's table, e.g. Galaxy's initialize (session open + platform seed) against a real gateway (#875).
|
||||
/// </param>
|
||||
public void Register(string driverType, Func<string, string, IDriver> factory,
|
||||
DriverTier tier = DriverTier.A)
|
||||
DriverTier tier = DriverTier.A,
|
||||
IReadOnlyDictionary<DriverCapability, CapabilityPolicy>? capabilityDefaults = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(driverType);
|
||||
ArgumentNullException.ThrowIfNull(factory);
|
||||
@@ -54,6 +63,7 @@ public sealed class DriverFactoryRegistry
|
||||
$"DriverType '{driverType}' factory already registered for this process");
|
||||
_factories[driverType] = factory;
|
||||
_tiers[driverType] = tier;
|
||||
if (capabilityDefaults is { Count: > 0 }) _capabilityDefaults[driverType] = capabilityDefaults;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,6 +93,15 @@ public sealed class DriverFactoryRegistry
|
||||
lock (_lock) return _tiers.GetValueOrDefault(driverType, DriverTier.A);
|
||||
}
|
||||
|
||||
/// <summary>The per-driver-type capability policy defaults registered with the factory, or null (#875).</summary>
|
||||
/// <param name="driverType">The driver type to look up.</param>
|
||||
/// <returns>The registered defaults, or <see langword="null"/> when the type registered none.</returns>
|
||||
public IReadOnlyDictionary<DriverCapability, CapabilityPolicy>? GetCapabilityDefaults(string driverType)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(driverType);
|
||||
lock (_lock) return _capabilityDefaults.GetValueOrDefault(driverType);
|
||||
}
|
||||
|
||||
/// <summary>Gets the collection of registered driver type names.</summary>
|
||||
public IReadOnlyCollection<string> RegisteredTypes
|
||||
{
|
||||
|
||||
@@ -34,6 +34,7 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
||||
private readonly Func<string, DriverTier> _tierResolver;
|
||||
private readonly ILogger? _logger;
|
||||
private readonly TimeSpan? _writeTimeoutCeiling;
|
||||
private readonly Func<string, IReadOnlyDictionary<DriverCapability, CapabilityPolicy>?>? _typeDefaultsResolver;
|
||||
private long _generation;
|
||||
|
||||
/// <summary>Construct the factory over the shared resilience infrastructure.</summary>
|
||||
@@ -46,12 +47,15 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
||||
/// cancels every write before the Write policy's own timeout can (lmxopcua#777). When non-null, a driver whose
|
||||
/// effective Write timeout exceeds it is logged as a warning at <see cref="Create"/> time: that policy timeout
|
||||
/// never takes effect. Null skips the check.</param>
|
||||
/// <param name="typeDefaultsResolver">Resolves a driver type's registered capability defaults
|
||||
/// (<c>DriverFactoryRegistry.GetCapabilityDefaults</c>, #875), layered over the tier table. Null = tier only.</param>
|
||||
public DriverCapabilityInvokerFactory(
|
||||
DriverResiliencePipelineBuilder builder,
|
||||
DriverResilienceStatusTracker statusTracker,
|
||||
Func<string, DriverTier> tierResolver,
|
||||
ILogger? logger = null,
|
||||
TimeSpan? writeTimeoutCeiling = null)
|
||||
TimeSpan? writeTimeoutCeiling = null,
|
||||
Func<string, IReadOnlyDictionary<DriverCapability, CapabilityPolicy>?>? typeDefaultsResolver = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(builder);
|
||||
ArgumentNullException.ThrowIfNull(statusTracker);
|
||||
@@ -61,13 +65,15 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
||||
_tierResolver = tierResolver;
|
||||
_logger = logger;
|
||||
_writeTimeoutCeiling = writeTimeoutCeiling;
|
||||
_typeDefaultsResolver = typeDefaultsResolver;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IDriverCapabilityInvoker Create(string driverInstanceId, string driverType, string? resilienceConfigJson = null)
|
||||
{
|
||||
var tier = _tierResolver(driverType);
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(tier, resilienceConfigJson, out var diagnostic);
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
tier, resilienceConfigJson, _typeDefaultsResolver?.Invoke(driverType), out var diagnostic);
|
||||
if (diagnostic is not null)
|
||||
{
|
||||
_logger?.LogWarning(
|
||||
|
||||
@@ -52,10 +52,32 @@ public static class DriverResilienceOptionsParser
|
||||
public static DriverResilienceOptions ParseOrDefaults(
|
||||
DriverTier tier,
|
||||
string? resilienceConfigJson,
|
||||
out string? parseDiagnostic) => ParseOrDefaults(tier, resilienceConfigJson, typeDefaults: null, out parseDiagnostic);
|
||||
|
||||
/// <summary>
|
||||
/// As <see cref="ParseOrDefaults(DriverTier, string?, out string?)"/>, with the driver TYPE's registered
|
||||
/// capability defaults (<c>DriverFactoryRegistry.GetCapabilityDefaults</c>, #875) layered over the tier
|
||||
/// table before the per-instance JSON is applied.
|
||||
/// </summary>
|
||||
/// <param name="tier">The driver's stability tier.</param>
|
||||
/// <param name="resilienceConfigJson">The per-instance ResilienceConfig JSON, or null.</param>
|
||||
/// <param name="typeDefaults">The driver type's capability defaults, or null.</param>
|
||||
/// <param name="parseDiagnostic">A diagnostic when the JSON was malformed or partly ignored.</param>
|
||||
/// <returns>The effective options.</returns>
|
||||
public static DriverResilienceOptions ParseOrDefaults(
|
||||
DriverTier tier,
|
||||
string? resilienceConfigJson,
|
||||
IReadOnlyDictionary<DriverCapability, CapabilityPolicy>? typeDefaults,
|
||||
out string? parseDiagnostic)
|
||||
{
|
||||
parseDiagnostic = null;
|
||||
var baseDefaults = DriverResilienceOptions.GetTierDefaults(tier);
|
||||
IReadOnlyDictionary<DriverCapability, CapabilityPolicy> baseDefaults = DriverResilienceOptions.GetTierDefaults(tier);
|
||||
if (typeDefaults is { Count: > 0 })
|
||||
{
|
||||
var layered = new Dictionary<DriverCapability, CapabilityPolicy>(baseDefaults);
|
||||
foreach (var (capability, policy) in typeDefaults) layered[capability] = policy;
|
||||
baseDefaults = layered;
|
||||
}
|
||||
var baseOptions = new DriverResilienceOptions { Tier = tier, CapabilityPolicies = baseDefaults };
|
||||
|
||||
if (string.IsNullOrWhiteSpace(resilienceConfigJson))
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -45,9 +46,23 @@ public static class GalaxyDriverFactoryExtensions
|
||||
ArgumentNullException.ThrowIfNull(registry);
|
||||
ArgumentNullException.ThrowIfNull(secretResolver);
|
||||
registry.Register(DriverTypeName,
|
||||
(id, json) => CreateInstance(id, json, loggerFactory, secretResolver, ackIdentityDirectory));
|
||||
(id, json) => CreateInstance(id, json, loggerFactory, secretResolver, ackIdentityDirectory),
|
||||
capabilityDefaults: CapabilityDefaults);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #875: Galaxy initialize (gateway session open + the per-template platform seed) took 4–5 s against the
|
||||
/// real VD03 gateway — right at the Tier A 5 s Initialize timeout, so a quarter of restarts failed and paid a
|
||||
/// 10 s retry plus a killed gateway worker. 20 s gives it 4× headroom and stays under the actor's 30 s
|
||||
/// pessimistic bound (<c>DriverInstanceActor.DefaultInitializeTimeout</c>). No retry and breaker threshold 2,
|
||||
/// as in every tier's Initialize policy. A per-instance ResilienceConfig still overrides it.
|
||||
/// </summary>
|
||||
internal static readonly IReadOnlyDictionary<DriverCapability, CapabilityPolicy> CapabilityDefaults =
|
||||
new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Initialize] = new(TimeoutSeconds: 20, RetryCount: 0, BreakerFailureThreshold: 2),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Convenience for tests + standalone callers. Uses the <see cref="NullSecretResolver"/>
|
||||
/// null-object, so the <c>secret:</c> API-key arm resolves fail-closed (absent) — callers
|
||||
|
||||
@@ -133,7 +133,8 @@ public static class DriverFactoryBootstrap
|
||||
// lmxopcua#777: warn at spawn when a per-instance ResilienceConfig Write timeout exceeds the
|
||||
// node's write timeout (it would never take effect). Unregistered ⇒ the default bounds.
|
||||
writeTimeoutCeiling: (sp.GetService<ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverWriteBounds>()
|
||||
?? ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverWriteBounds.Default).Timeout);
|
||||
?? ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverWriteBounds.Default).Timeout,
|
||||
typeDefaultsResolver: registry.GetCapabilityDefaults);
|
||||
});
|
||||
|
||||
// The tracker's operator-facing reader: a periodic DPS publisher that mirrors each snapshot to the
|
||||
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Hosting;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
/// <summary>
|
||||
/// #875: a driver TYPE can register capability defaults that layer over its tier table and under the
|
||||
/// per-instance ResilienceConfig — Galaxy's initialize did not fit Tier A's 5 s.
|
||||
/// </summary>
|
||||
public sealed class DriverTypeCapabilityDefaultsTests
|
||||
{
|
||||
private static readonly Dictionary<DriverCapability, CapabilityPolicy> TypeDefaults = new()
|
||||
{
|
||||
[DriverCapability.Initialize] = new(TimeoutSeconds: 20, RetryCount: 0, BreakerFailureThreshold: 2),
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Type_defaults_override_the_tier_for_their_capability_only()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, null, TypeDefaults, out var diag);
|
||||
diag.ShouldBeNull();
|
||||
options.Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(20);
|
||||
options.Resolve(DriverCapability.Read).ShouldBe(DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_per_instance_ResilienceConfig_still_overrides_the_type_default()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, """{"capabilityPolicies":{"Initialize":{"timeoutSeconds":12}}}""", TypeDefaults, out _);
|
||||
options.Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(12);
|
||||
options.Resolve(DriverCapability.Initialize).BreakerFailureThreshold.ShouldBe(2, "unspecified fields come from the type default");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Without_type_defaults_the_tier_table_is_unchanged()
|
||||
{
|
||||
DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, null, null, out _)
|
||||
.Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_registry_returns_what_a_type_registered_and_null_otherwise()
|
||||
{
|
||||
var registry = new DriverFactoryRegistry();
|
||||
registry.Register("WithDefaults", (_, _) => null!, capabilityDefaults: TypeDefaults);
|
||||
registry.Register("Plain", (_, _) => null!);
|
||||
registry.GetCapabilityDefaults("withdefaults")!.ShouldContainKey(DriverCapability.Initialize);
|
||||
registry.GetCapabilityDefaults("Plain").ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_invoker_factory_applies_the_resolved_type_defaults()
|
||||
{
|
||||
var registry = new DriverFactoryRegistry();
|
||||
registry.Register("Slow", (_, _) => null!, capabilityDefaults: TypeDefaults);
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
var factory = new DriverCapabilityInvokerFactory(new DriverResiliencePipelineBuilder(statusTracker: tracker), tracker,
|
||||
registry.GetTier, typeDefaultsResolver: registry.GetCapabilityDefaults);
|
||||
var invoker = factory.Create("slow-1", "Slow");
|
||||
// The options the invoker's pipelines are built from (private accessor; no public surface exposes them).
|
||||
var accessor = (Func<DriverResilienceOptions>)invoker.GetType()
|
||||
.GetField("_optionsAccessor", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!
|
||||
.GetValue(invoker)!;
|
||||
accessor().Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(20);
|
||||
accessor().Resolve(DriverCapability.Read).TimeoutSeconds.ShouldBe(2);
|
||||
}
|
||||
}
|
||||
@@ -351,4 +351,15 @@ public sealed class GalaxyDriverFactoryTests
|
||||
var driver = GalaxyDriverFactoryExtensions.CreateInstance("g", MinimalConfig);
|
||||
driver.DriverType.ShouldBe("GalaxyMxGateway");
|
||||
}
|
||||
|
||||
/// <summary>#875: the Galaxy registration carries a 20 s Initialize budget over Tier A's 5 s.</summary>
|
||||
[Fact]
|
||||
public void Register_carries_the_Galaxy_initialize_budget()
|
||||
{
|
||||
var registry = new ZB.MOM.WW.OtOpcUa.Core.Hosting.DriverFactoryRegistry();
|
||||
GalaxyDriverFactoryExtensions.Register(registry, ZB.MOM.WW.OtOpcUa.Driver.Galaxy.NullSecretResolver.Instance);
|
||||
var defaults = registry.GetCapabilityDefaults(GalaxyDriverFactoryExtensions.DriverTypeName);
|
||||
defaults.ShouldNotBeNull();
|
||||
defaults![ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverCapability.Initialize].TimeoutSeconds.ShouldBe(20);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user