fix(galaxy): #875 Galaxy initialize gets a 20 s budget via per-driver-type capability defaults

Gate F (#761) against the VD03 gateway: Galaxy initialize (session open +
platform seed) took 4–5 s against Tier A's 5 s Initialize timeout, so 10 of 40
restarts failed, paid a 10 s retry and left a killed gateway worker.

DriverFactoryRegistry.Register takes optional capabilityDefaults per driver
type (GetCapabilityDefaults); DriverResilienceOptionsParser layers them over the
tier table and under the per-instance ResilienceConfig; the Host wires the
registry into DriverCapabilityInvokerFactory. Galaxy registers Initialize =
20 s (no retry, breaker 2), under the actor's 30 s pessimistic bound.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 18:44:44 -05:00
parent 2be43aa984
commit 95bde828eb
8 changed files with 157 additions and 7 deletions
+6 -1
View File
@@ -134,7 +134,12 @@ following this note. `DriverResilienceOptions.GetTierDefaults(tier)` supplies
the real per-capability timeout / retry / breaker policies, resolved via
`DriverFactoryRegistry.GetTier` and applied by `DriverCapabilityInvokerFactory`.
Every driver runs Tier A because no factory passes a tier, so today that means
one uniform policy set.
one uniform policy set — except where a driver type registers **capability
defaults** with its factory (`DriverFactoryRegistry.Register(...,
capabilityDefaults:)`, read back by `GetCapabilityDefaults`). Those layer over
the tier table and under any per-instance `ResilienceConfig`. Galaxy registers
`Initialize` = 20 s (#875): its session open plus platform seed took 4–5 s
against a real gateway, so Tier A's 5 s failed a quarter of restarts.
`IDriver` **no longer carries `GetMemoryFootprint()` or
`FlushOptionalCachesAsync()`** (Gitea **#525**). They existed to feed
@@ -1,3 +1,4 @@
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Core.Hosting;
@@ -21,6 +22,8 @@ public sealed class DriverFactoryRegistry
= new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, DriverTier> _tiers
= new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, IReadOnlyDictionary<DriverCapability, CapabilityPolicy>> _capabilityDefaults
= new(StringComparer.OrdinalIgnoreCase);
private readonly object _lock = new();
/// <summary>
@@ -42,8 +45,14 @@ public sealed class DriverFactoryRegistry
/// <c>DriverInstanceBootstrapper</c> can skip in-process drivers by tier check
/// rather than by driver-type allow-list.
/// </param>
/// <param name="capabilityDefaults">
/// Optional per-driver-TYPE capability policies layered over the tier defaults (and under any
/// per-instance <c>ResilienceConfig</c>) — for a driver whose real cost of one capability does not fit its
/// tier's table, e.g. Galaxy's initialize (session open + platform seed) against a real gateway (#875).
/// </param>
public void Register(string driverType, Func<string, string, IDriver> factory,
DriverTier tier = DriverTier.A)
DriverTier tier = DriverTier.A,
IReadOnlyDictionary<DriverCapability, CapabilityPolicy>? capabilityDefaults = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(driverType);
ArgumentNullException.ThrowIfNull(factory);
@@ -54,6 +63,7 @@ public sealed class DriverFactoryRegistry
$"DriverType '{driverType}' factory already registered for this process");
_factories[driverType] = factory;
_tiers[driverType] = tier;
if (capabilityDefaults is { Count: > 0 }) _capabilityDefaults[driverType] = capabilityDefaults;
}
}
@@ -83,6 +93,15 @@ public sealed class DriverFactoryRegistry
lock (_lock) return _tiers.GetValueOrDefault(driverType, DriverTier.A);
}
/// <summary>The per-driver-type capability policy defaults registered with the factory, or null (#875).</summary>
/// <param name="driverType">The driver type to look up.</param>
/// <returns>The registered defaults, or <see langword="null"/> when the type registered none.</returns>
public IReadOnlyDictionary<DriverCapability, CapabilityPolicy>? GetCapabilityDefaults(string driverType)
{
ArgumentException.ThrowIfNullOrWhiteSpace(driverType);
lock (_lock) return _capabilityDefaults.GetValueOrDefault(driverType);
}
/// <summary>Gets the collection of registered driver type names.</summary>
public IReadOnlyCollection<string> RegisteredTypes
{
@@ -34,6 +34,7 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
private readonly Func<string, DriverTier> _tierResolver;
private readonly ILogger? _logger;
private readonly TimeSpan? _writeTimeoutCeiling;
private readonly Func<string, IReadOnlyDictionary<DriverCapability, CapabilityPolicy>?>? _typeDefaultsResolver;
private long _generation;
/// <summary>Construct the factory over the shared resilience infrastructure.</summary>
@@ -46,12 +47,15 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
/// cancels every write before the Write policy's own timeout can (lmxopcua#777). When non-null, a driver whose
/// effective Write timeout exceeds it is logged as a warning at <see cref="Create"/> time: that policy timeout
/// never takes effect. Null skips the check.</param>
/// <param name="typeDefaultsResolver">Resolves a driver type's registered capability defaults
/// (<c>DriverFactoryRegistry.GetCapabilityDefaults</c>, #875), layered over the tier table. Null = tier only.</param>
public DriverCapabilityInvokerFactory(
DriverResiliencePipelineBuilder builder,
DriverResilienceStatusTracker statusTracker,
Func<string, DriverTier> tierResolver,
ILogger? logger = null,
TimeSpan? writeTimeoutCeiling = null)
TimeSpan? writeTimeoutCeiling = null,
Func<string, IReadOnlyDictionary<DriverCapability, CapabilityPolicy>?>? typeDefaultsResolver = null)
{
ArgumentNullException.ThrowIfNull(builder);
ArgumentNullException.ThrowIfNull(statusTracker);
@@ -61,13 +65,15 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
_tierResolver = tierResolver;
_logger = logger;
_writeTimeoutCeiling = writeTimeoutCeiling;
_typeDefaultsResolver = typeDefaultsResolver;
}
/// <inheritdoc />
public IDriverCapabilityInvoker Create(string driverInstanceId, string driverType, string? resilienceConfigJson = null)
{
var tier = _tierResolver(driverType);
var options = DriverResilienceOptionsParser.ParseOrDefaults(tier, resilienceConfigJson, out var diagnostic);
var options = DriverResilienceOptionsParser.ParseOrDefaults(
tier, resilienceConfigJson, _typeDefaultsResolver?.Invoke(driverType), out var diagnostic);
if (diagnostic is not null)
{
_logger?.LogWarning(
@@ -52,10 +52,32 @@ public static class DriverResilienceOptionsParser
public static DriverResilienceOptions ParseOrDefaults(
DriverTier tier,
string? resilienceConfigJson,
out string? parseDiagnostic) => ParseOrDefaults(tier, resilienceConfigJson, typeDefaults: null, out parseDiagnostic);
/// <summary>
/// As <see cref="ParseOrDefaults(DriverTier, string?, out string?)"/>, with the driver TYPE's registered
/// capability defaults (<c>DriverFactoryRegistry.GetCapabilityDefaults</c>, #875) layered over the tier
/// table before the per-instance JSON is applied.
/// </summary>
/// <param name="tier">The driver's stability tier.</param>
/// <param name="resilienceConfigJson">The per-instance ResilienceConfig JSON, or null.</param>
/// <param name="typeDefaults">The driver type's capability defaults, or null.</param>
/// <param name="parseDiagnostic">A diagnostic when the JSON was malformed or partly ignored.</param>
/// <returns>The effective options.</returns>
public static DriverResilienceOptions ParseOrDefaults(
DriverTier tier,
string? resilienceConfigJson,
IReadOnlyDictionary<DriverCapability, CapabilityPolicy>? typeDefaults,
out string? parseDiagnostic)
{
parseDiagnostic = null;
var baseDefaults = DriverResilienceOptions.GetTierDefaults(tier);
IReadOnlyDictionary<DriverCapability, CapabilityPolicy> baseDefaults = DriverResilienceOptions.GetTierDefaults(tier);
if (typeDefaults is { Count: > 0 })
{
var layered = new Dictionary<DriverCapability, CapabilityPolicy>(baseDefaults);
foreach (var (capability, policy) in typeDefaults) layered[capability] = policy;
baseDefaults = layered;
}
var baseOptions = new DriverResilienceOptions { Tier = tier, CapabilityPolicies = baseDefaults };
if (string.IsNullOrWhiteSpace(resilienceConfigJson))
@@ -1,3 +1,4 @@
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
@@ -45,9 +46,23 @@ public static class GalaxyDriverFactoryExtensions
ArgumentNullException.ThrowIfNull(registry);
ArgumentNullException.ThrowIfNull(secretResolver);
registry.Register(DriverTypeName,
(id, json) => CreateInstance(id, json, loggerFactory, secretResolver, ackIdentityDirectory));
(id, json) => CreateInstance(id, json, loggerFactory, secretResolver, ackIdentityDirectory),
capabilityDefaults: CapabilityDefaults);
}
/// <summary>
/// #875: Galaxy initialize (gateway session open + the per-template platform seed) took 4–5 s against the
/// real VD03 gateway — right at the Tier A 5 s Initialize timeout, so a quarter of restarts failed and paid a
/// 10 s retry plus a killed gateway worker. 20 s gives it 4× headroom and stays under the actor's 30 s
/// pessimistic bound (<c>DriverInstanceActor.DefaultInitializeTimeout</c>). No retry and breaker threshold 2,
/// as in every tier's Initialize policy. A per-instance ResilienceConfig still overrides it.
/// </summary>
internal static readonly IReadOnlyDictionary<DriverCapability, CapabilityPolicy> CapabilityDefaults =
new Dictionary<DriverCapability, CapabilityPolicy>
{
[DriverCapability.Initialize] = new(TimeoutSeconds: 20, RetryCount: 0, BreakerFailureThreshold: 2),
};
/// <summary>
/// Convenience for tests + standalone callers. Uses the <see cref="NullSecretResolver"/>
/// null-object, so the <c>secret:</c> API-key arm resolves fail-closed (absent) — callers
@@ -133,7 +133,8 @@ public static class DriverFactoryBootstrap
// lmxopcua#777: warn at spawn when a per-instance ResilienceConfig Write timeout exceeds the
// node's write timeout (it would never take effect). Unregistered ⇒ the default bounds.
writeTimeoutCeiling: (sp.GetService<ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverWriteBounds>()
?? ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverWriteBounds.Default).Timeout);
?? ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverWriteBounds.Default).Timeout,
typeDefaultsResolver: registry.GetCapabilityDefaults);
});
// The tracker's operator-facing reader: a periodic DPS publisher that mirrors each snapshot to the
@@ -0,0 +1,71 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Hosting;
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
/// <summary>
/// #875: a driver TYPE can register capability defaults that layer over its tier table and under the
/// per-instance ResilienceConfig — Galaxy's initialize did not fit Tier A's 5 s.
/// </summary>
public sealed class DriverTypeCapabilityDefaultsTests
{
private static readonly Dictionary<DriverCapability, CapabilityPolicy> TypeDefaults = new()
{
[DriverCapability.Initialize] = new(TimeoutSeconds: 20, RetryCount: 0, BreakerFailureThreshold: 2),
};
[Fact]
public void Type_defaults_override_the_tier_for_their_capability_only()
{
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, null, TypeDefaults, out var diag);
diag.ShouldBeNull();
options.Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(20);
options.Resolve(DriverCapability.Read).ShouldBe(DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read]);
}
[Fact]
public void A_per_instance_ResilienceConfig_still_overrides_the_type_default()
{
var options = DriverResilienceOptionsParser.ParseOrDefaults(
DriverTier.A, """{"capabilityPolicies":{"Initialize":{"timeoutSeconds":12}}}""", TypeDefaults, out _);
options.Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(12);
options.Resolve(DriverCapability.Initialize).BreakerFailureThreshold.ShouldBe(2, "unspecified fields come from the type default");
}
[Fact]
public void Without_type_defaults_the_tier_table_is_unchanged()
{
DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, null, null, out _)
.Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(5);
}
[Fact]
public void The_registry_returns_what_a_type_registered_and_null_otherwise()
{
var registry = new DriverFactoryRegistry();
registry.Register("WithDefaults", (_, _) => null!, capabilityDefaults: TypeDefaults);
registry.Register("Plain", (_, _) => null!);
registry.GetCapabilityDefaults("withdefaults")!.ShouldContainKey(DriverCapability.Initialize);
registry.GetCapabilityDefaults("Plain").ShouldBeNull();
}
[Fact]
public void The_invoker_factory_applies_the_resolved_type_defaults()
{
var registry = new DriverFactoryRegistry();
registry.Register("Slow", (_, _) => null!, capabilityDefaults: TypeDefaults);
var tracker = new DriverResilienceStatusTracker();
var factory = new DriverCapabilityInvokerFactory(new DriverResiliencePipelineBuilder(statusTracker: tracker), tracker,
registry.GetTier, typeDefaultsResolver: registry.GetCapabilityDefaults);
var invoker = factory.Create("slow-1", "Slow");
// The options the invoker's pipelines are built from (private accessor; no public surface exposes them).
var accessor = (Func<DriverResilienceOptions>)invoker.GetType()
.GetField("_optionsAccessor", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!
.GetValue(invoker)!;
accessor().Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(20);
accessor().Resolve(DriverCapability.Read).TimeoutSeconds.ShouldBe(2);
}
}
@@ -351,4 +351,15 @@ public sealed class GalaxyDriverFactoryTests
var driver = GalaxyDriverFactoryExtensions.CreateInstance("g", MinimalConfig);
driver.DriverType.ShouldBe("GalaxyMxGateway");
}
/// <summary>#875: the Galaxy registration carries a 20 s Initialize budget over Tier A's 5 s.</summary>
[Fact]
public void Register_carries_the_Galaxy_initialize_budget()
{
var registry = new ZB.MOM.WW.OtOpcUa.Core.Hosting.DriverFactoryRegistry();
GalaxyDriverFactoryExtensions.Register(registry, ZB.MOM.WW.OtOpcUa.Driver.Galaxy.NullSecretResolver.Instance);
var defaults = registry.GetCapabilityDefaults(GalaxyDriverFactoryExtensions.DriverTypeName);
defaults.ShouldNotBeNull();
defaults![ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverCapability.Initialize].TimeoutSeconds.ShouldBe(20);
}
}