feat(archreview #13): apply per-instance ResilienceConfig from the deploy artifact
The DriverInstance.ResilienceConfig column was authored in AdminUI, persisted to the entity, and serialized into the deployment artifact by ConfigComposer — but the runtime read path dropped it: DriverInstanceSpec didn't carry it and the invoker factory always passed null, so every driver got tier defaults regardless of its configured overrides (a silent dead-config gap — #10's residual sub-finding). Read-path plumbing (write side was already complete): - DriverInstanceSpec gains ResilienceConfig; DeploymentArtifact.TryReadSpec reads the column. - IDriverCapabilityInvokerFactory.Create takes resilienceConfigJson; DriverHostActor.SpawnChild threads spec.ResilienceConfig; the concrete factory parses it (ParseOrDefaults, layering on the tier), logs any parse diagnostic (never throws), and builds the invoker with the merged options. - Invalidate-on-change: the pipeline cache keys on (instance, host, capability) and ignores options on a hit, so Create() now Invalidate()s the instance's cached pipelines first (no-op on first spawn) — a respawn with changed options rebuilds them. - DriverSpawnPlanner treats a ResilienceConfig change as a stop+respawn (the invoker/options are bound to the child at spawn); a pure DriverConfig change stays an in-place delta (no reconnect). - Host DI passes a logger to the factory for the parse diagnostic. Verification (deterministic): factory Create applies a retryCount:0 override to actual execution (control test proves tier default retries), invalidates the instance's cache on re-create (scoped — sibling survives), malformed config logs+falls-back; planner respawns on ResilienceConfig change (incl null→json) and stays delta on a pure config change; artifact parse carries/omits the column. Core.Tests 243 (+5), Runtime.Tests 363 (+5), Host builds clean.
This commit is contained in:
Joseph Doherty
@@ -14,12 +14,20 @@ public interface IDriverCapabilityInvokerFactory
|
||||
/// <summary>
|
||||
/// Create the resilience invoker for one driver instance. The returned invoker is keyed on
|
||||
/// <paramref name="driverInstanceId"/> and applies the tier policy resolved from
|
||||
/// <paramref name="driverType"/>.
|
||||
/// <paramref name="driverType"/>, layering any per-instance
|
||||
/// <paramref name="resilienceConfigJson"/> overrides on top.
|
||||
/// </summary>
|
||||
/// <param name="driverInstanceId">The driver instance identifier (pipeline + tracker key).</param>
|
||||
/// <param name="driverType">The driver type name, used to resolve the resilience tier defaults.</param>
|
||||
/// <param name="resilienceConfigJson">
|
||||
/// Optional per-instance <c>DriverInstance.ResilienceConfig</c> JSON (from the deployment artifact)
|
||||
/// layered on top of the tier defaults. <c>null</c>/empty ⇒ pure tier defaults. A concrete factory
|
||||
/// invalidates any cached pipelines for <paramref name="driverInstanceId"/> so a changed config
|
||||
/// applied across a driver respawn takes effect (the pipeline cache is keyed on instance/host/
|
||||
/// capability, not options).
|
||||
/// </param>
|
||||
/// <returns>A per-instance <see cref="IDriverCapabilityInvoker"/>.</returns>
|
||||
IDriverCapabilityInvoker Create(string driverInstanceId, string driverType);
|
||||
IDriverCapabilityInvoker Create(string driverInstanceId, string driverType, string? resilienceConfigJson = null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -34,6 +42,6 @@ public sealed class NullDriverCapabilityInvokerFactory : IDriverCapabilityInvoke
|
||||
private NullDriverCapabilityInvokerFactory() { }
|
||||
|
||||
/// <inheritdoc />
|
||||
public IDriverCapabilityInvoker Create(string driverInstanceId, string driverType) =>
|
||||
public IDriverCapabilityInvoker Create(string driverInstanceId, string driverType, string? resilienceConfigJson = null) =>
|
||||
NullDriverCapabilityInvoker.Instance;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
@@ -9,30 +10,39 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
/// resolves the abstraction from DI and never sees this type.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>Resilience policy is currently the <b>tier default</b> for the driver type — resolved
|
||||
/// via the injected tier resolver (backed by <c>DriverFactoryRegistry.GetTier</c>). The
|
||||
/// per-instance <c>DriverInstance.ResilienceConfig</c> JSON override is NOT yet applied because
|
||||
/// the deployment artifact (<c>DriverInstanceSpec</c>) does not carry it; plumbing that column
|
||||
/// through the composer/artifact is a tracked follow-up. Until then every instance of a given
|
||||
/// driver type gets that type's tier defaults.</para>
|
||||
/// <para>Resilience policy is the driver type's <b>tier default</b> (resolved via the injected
|
||||
/// tier resolver, backed by <c>DriverFactoryRegistry.GetTier</c>) with the per-instance
|
||||
/// <c>DriverInstance.ResilienceConfig</c> JSON <b>layered on top</b> when the deployment artifact
|
||||
/// carries one (<c>DriverInstanceSpec.ResilienceConfig</c>, threaded in by the Runtime
|
||||
/// <c>DriverHostActor</c> at spawn). A missing/empty/malformed config degrades to pure tier
|
||||
/// defaults — a bad ResilienceConfig never bricks a driver.</para>
|
||||
///
|
||||
/// <para>Options are snapshotted once per <see cref="Create"/> (tier is fixed for a driver type),
|
||||
/// so the invoker's per-call options accessor is allocation-free on the hot path.</para>
|
||||
/// <para>Options are snapshotted once per <see cref="Create"/> (a driver actor's options are fixed
|
||||
/// for its lifetime — a ResilienceConfig change respawns the child), so the invoker's per-call
|
||||
/// options accessor is allocation-free on the hot path. Because the pipeline builder caches
|
||||
/// pipelines keyed on <c>(instance, host, capability)</c> and <b>ignores options on a cache hit</b>,
|
||||
/// <see cref="Create"/> first <see cref="DriverResiliencePipelineBuilder.Invalidate"/>s any pipelines
|
||||
/// cached for the instance so a respawn with changed options rebuilds them (rather than silently
|
||||
/// reusing the stale pipeline). On a first spawn this removes nothing.</para>
|
||||
/// </remarks>
|
||||
public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFactory
|
||||
{
|
||||
private readonly DriverResiliencePipelineBuilder _builder;
|
||||
private readonly DriverResilienceStatusTracker _statusTracker;
|
||||
private readonly Func<string, DriverTier> _tierResolver;
|
||||
private readonly ILogger? _logger;
|
||||
|
||||
/// <summary>Construct the factory over the shared resilience infrastructure.</summary>
|
||||
/// <param name="builder">Process-singleton Polly pipeline builder (shared pipeline cache).</param>
|
||||
/// <param name="statusTracker">Process-singleton resilience status tracker feeding Admin <c>/hosts</c>.</param>
|
||||
/// <param name="tierResolver">Resolves a driver type name to its <see cref="DriverTier"/> (e.g. <c>DriverFactoryRegistry.GetTier</c>).</param>
|
||||
/// <param name="logger">Optional logger; when non-null, a ResilienceConfig parse diagnostic (malformed JSON,
|
||||
/// unknown capability, misapplied Tier-C recycle) is logged as a warning at <see cref="Create"/> time.</param>
|
||||
public DriverCapabilityInvokerFactory(
|
||||
DriverResiliencePipelineBuilder builder,
|
||||
DriverResilienceStatusTracker statusTracker,
|
||||
Func<string, DriverTier> tierResolver)
|
||||
Func<string, DriverTier> tierResolver,
|
||||
ILogger? logger = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(builder);
|
||||
ArgumentNullException.ThrowIfNull(statusTracker);
|
||||
@@ -40,14 +50,26 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
||||
_builder = builder;
|
||||
_statusTracker = statusTracker;
|
||||
_tierResolver = tierResolver;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IDriverCapabilityInvoker Create(string driverInstanceId, string driverType)
|
||||
public IDriverCapabilityInvoker Create(string driverInstanceId, string driverType, string? resilienceConfigJson = null)
|
||||
{
|
||||
var tier = _tierResolver(driverType);
|
||||
// No per-instance ResilienceConfig in the artifact yet — tier defaults only (see remarks).
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(tier, resilienceConfigJson: null, out _);
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(tier, resilienceConfigJson, out var diagnostic);
|
||||
if (diagnostic is not null)
|
||||
{
|
||||
_logger?.LogWarning(
|
||||
"Driver resilience config for instance={DriverInstanceId} type={DriverType}: {Diagnostic}",
|
||||
driverInstanceId, driverType, diagnostic);
|
||||
}
|
||||
|
||||
// Drop any pipelines cached for this instance under the PREVIOUS options — the builder keys on
|
||||
// (instance, host, capability) and reuses a cached pipeline regardless of options, so a respawn
|
||||
// with a changed ResilienceConfig would otherwise keep serving the stale pipeline. No-op on first spawn.
|
||||
_builder.Invalidate(driverInstanceId);
|
||||
|
||||
return new CapabilityInvoker(
|
||||
_builder,
|
||||
driverInstanceId,
|
||||
|
||||
@@ -66,7 +66,8 @@ public static class DriverFactoryBootstrap
|
||||
return new DriverCapabilityInvokerFactory(
|
||||
sp.GetRequiredService<DriverResiliencePipelineBuilder>(),
|
||||
sp.GetRequiredService<DriverResilienceStatusTracker>(),
|
||||
registry.GetTier);
|
||||
registry.GetTier,
|
||||
logger: sp.GetService<ILoggerFactory>()?.CreateLogger("ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverCapabilityInvokerFactory"));
|
||||
});
|
||||
|
||||
// Driver nodes also carry the probe set so a fused admin,driver node has it; the admin-only
|
||||
|
||||
@@ -18,7 +18,8 @@ public sealed record DriverInstanceSpec(
|
||||
string DriverType,
|
||||
bool Enabled,
|
||||
string DriverConfig,
|
||||
string? ClusterId = null);
|
||||
string? ClusterId = null,
|
||||
string? ResilienceConfig = null);
|
||||
|
||||
/// <summary>How a node should scope a deployment artifact to its own ClusterId.</summary>
|
||||
public enum ClusterFilterMode
|
||||
@@ -167,6 +168,9 @@ public static class DeploymentArtifact
|
||||
|| enEl.GetBoolean();
|
||||
var config = ReadString(el, "DriverConfig");
|
||||
var clusterId = ReadString(el, "ClusterId");
|
||||
// Per-instance resilience overrides (DriverInstance.ResilienceConfig column). Optional/nullable —
|
||||
// absent means tier defaults; layered onto the tier by DriverResilienceOptionsParser at spawn.
|
||||
var resilienceConfig = ReadString(el, "ResilienceConfig");
|
||||
|
||||
if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(type)) return null;
|
||||
|
||||
@@ -177,7 +181,8 @@ public static class DeploymentArtifact
|
||||
DriverType: type!,
|
||||
Enabled: enabled,
|
||||
DriverConfig: config ?? "{}",
|
||||
ClusterId: clusterId);
|
||||
ClusterId: clusterId,
|
||||
ResilienceConfig: resilienceConfig);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -244,6 +244,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
public string DriverType => Spec.DriverType;
|
||||
/// <summary>Gets the last applied driver configuration JSON from the child's spec.</summary>
|
||||
public string LastConfigJson => Spec.DriverConfig;
|
||||
/// <summary>Gets the per-instance ResilienceConfig JSON the child's invoker was built with (null = tier defaults).</summary>
|
||||
public string? ResilienceConfig => Spec.ResilienceConfig;
|
||||
}
|
||||
|
||||
/// <summary>Gets or sets the timer scheduler for this actor's config-db retry and rediscovery timers.</summary>
|
||||
@@ -1269,7 +1271,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
var specs = DeploymentArtifact.ParseDriverInstances(blob, _localNode.Value);
|
||||
var snapshots = _children.ToDictionary(
|
||||
kv => kv.Key,
|
||||
kv => new DriverChildSnapshot(kv.Value.DriverType, kv.Value.LastConfigJson),
|
||||
kv => new DriverChildSnapshot(kv.Value.DriverType, kv.Value.LastConfigJson, kv.Value.ResilienceConfig),
|
||||
StringComparer.Ordinal);
|
||||
var plan = DriverSpawnPlanner.Compute(snapshots, specs);
|
||||
|
||||
@@ -1625,9 +1627,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
else
|
||||
{
|
||||
// Attach the per-instance Phase 6.1 resilience invoker so this driver's capability calls
|
||||
// route through the retry/breaker/bulkhead/telemetry pipeline. The pass-through factory
|
||||
// yields a no-op invoker on nodes without the real resilience factory bound.
|
||||
var invoker = _invokerFactory.Create(spec.DriverInstanceId, spec.DriverType);
|
||||
// route through the retry/breaker/bulkhead/telemetry pipeline. The tier defaults are layered
|
||||
// with the per-instance ResilienceConfig from the artifact; the pass-through factory yields a
|
||||
// no-op invoker on nodes without the real resilience factory bound.
|
||||
var invoker = _invokerFactory.Create(spec.DriverInstanceId, spec.DriverType, spec.ResilienceConfig);
|
||||
child = Context.ActorOf(
|
||||
DriverInstanceActor.Props(
|
||||
driver!,
|
||||
|
||||
@@ -42,8 +42,14 @@ public static class DriverSpawnPlanner
|
||||
toStop.Add(id);
|
||||
continue;
|
||||
}
|
||||
// Driver type changes can't be reinitialized in-place (factory-bound) — stop + respawn.
|
||||
if (!string.Equals(snap.DriverType, spec.DriverType, StringComparison.Ordinal))
|
||||
// A driver TYPE change can't be reinitialized in-place (factory-bound) — stop + respawn.
|
||||
// A RESILIENCE-CONFIG change likewise forces a respawn: the CapabilityInvoker (and its
|
||||
// resolved options) is bound to the child actor at spawn time, so the only way a changed
|
||||
// ResilienceConfig takes effect is to rebuild the child. The factory invalidates the stale
|
||||
// cached pipelines on the respawn's Create call. (A pure DriverConfig change stays an
|
||||
// in-place delta — no reconnect — because it doesn't touch the resilience pipeline.)
|
||||
if (!string.Equals(snap.DriverType, spec.DriverType, StringComparison.Ordinal)
|
||||
|| !string.Equals(snap.ResilienceConfig, spec.ResilienceConfig, StringComparison.Ordinal))
|
||||
{
|
||||
toStop.Add(id);
|
||||
toSpawn.Add(spec);
|
||||
@@ -67,4 +73,7 @@ public static class DriverSpawnPlanner
|
||||
}
|
||||
|
||||
/// <summary>Snapshot of one running driver child as the host sees it. Used as the diff input.</summary>
|
||||
public sealed record DriverChildSnapshot(string DriverType, string LastConfigJson);
|
||||
/// <param name="DriverType">The driver type name the child was spawned for.</param>
|
||||
/// <param name="LastConfigJson">The <c>DriverConfig</c> JSON the child currently serves.</param>
|
||||
/// <param name="ResilienceConfig">The per-instance <c>ResilienceConfig</c> JSON the child's invoker was built with (null = tier defaults).</param>
|
||||
public sealed record DriverChildSnapshot(string DriverType, string LastConfigJson, string? ResilienceConfig = null);
|
||||
|
||||
Reference in New Issue
Block a user