Merge R2-02 ResilienceConfig hardening (arch-review round 2) [PR #431]
v2-ci / build (push) Successful in 3m53s
v2-ci / unit-tests (push) Failing after 8m15s

Findings 01/S-6 (High — clamp hostile ResilienceConfig, no ValidationException
driver-brick), 01/U-6 (delete dead bulkhead knob), 01/S-8=03/S12 (Write/Ack
non-idempotent + retry->0), 04/C-7 (JsonObject round-trip preserves unknown
keys), 01/S-7=03/S13 (per-Create options-generation kills respawn cache race).
T15/T18 deferred. STATUS.md conflict resolved additively. Build clean.
This commit is contained in:
Joseph Doherty
2026-07-13 10:29:43 -04:00
36 changed files with 1016 additions and 197 deletions
@@ -35,8 +35,6 @@ public sealed class DriverInstance
/// Null = use the driver's tier defaults. When populated, expected shape:
/// <code>
/// {
/// "bulkheadMaxConcurrent": 16,
/// "bulkheadMaxQueue": 64,
/// "capabilityPolicies": {
/// "Read": { "timeoutSeconds": 5, "retryCount": 5, "breakerFailureThreshold": 3 },
/// "Write": { "timeoutSeconds": 5, "retryCount": 0, "breakerFailureThreshold": 5 }
@@ -26,7 +26,9 @@ public sealed class DriverInstanceResilienceStatus
/// <summary>Rolling count of consecutive Polly pipeline failures for this (instance, host).</summary>
public int ConsecutiveFailures { get; set; }
/// <summary>Current Polly bulkhead depth (in-flight calls) for this (instance, host).</summary>
/// <summary>In-flight capability-call depth for this (instance, host). Formerly fed a Polly
/// concurrency-limiter strategy that was removed (R2-02/01/U-6); the column name is retained to avoid a
/// migration on a reader-less entity — it now carries the tracker's in-flight-call count.</summary>
public int CurrentBulkheadDepth { get; set; }
/// <summary>Most recent process recycle time (Tier C only; null for in-process tiers).</summary>
@@ -2,7 +2,7 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
/// <summary>
/// Abstraction over the Phase 6.1 resilience pipeline that wraps a single driver instance's
/// capability calls (retry / circuit-breaker / bulkhead / tracker telemetry). The Runtime
/// capability calls (retry / circuit-breaker / in-flight accounting / tracker telemetry). The Runtime
/// dispatch layer (<c>DriverInstanceActor</c>) consumes this interface instead of the concrete
/// <c>CapabilityInvoker</c> so the actor assembly does NOT pull in
/// <c>ZB.MOM.WW.OtOpcUa.Core</c> (which drags in Polly + driver hosting) — the same boundary
@@ -15,7 +15,7 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
/// this interface without adapters. Callers pass the resolved <c>hostName</c> per call (a
/// multi-host driver resolves it from the tag reference via <see cref="IPerCallHostResolver"/>;
/// a single-host driver passes its <see cref="IDriver.DriverInstanceId"/>) so per-host breaker
/// isolation + bulkhead-depth accounting key on the right target.
/// isolation + in-flight-depth accounting key on the right target.
/// </remarks>
public interface IDriverCapabilityInvoker
{
@@ -48,7 +48,10 @@ public interface IDriverCapabilityInvoker
/// Execute a <see cref="DriverCapability.Write"/> call honoring idempotence semantics — when
/// <paramref name="isIdempotent"/> is <c>false</c>, retries are disabled regardless of the
/// configured policy; when <c>true</c>, the call runs through the Write pipeline which may
/// retry if the tier configuration permits.
/// retry if the tier configuration permits. <b>The current production default is <c>false</c></b>
/// (R2-02/S-8): <c>DriverInstanceActor.HandleWriteAsync</c> passes <c>isIdempotent: false</c> so a
/// command-shaped write is never replayed; per-tag opt-in via <see cref="WriteIdempotentAttribute"/>
/// is the unshipped forward path.
/// </summary>
/// <typeparam name="TResult">Return type of the underlying driver write call.</typeparam>
/// <param name="hostName">The resolved host name for pipeline keying + status tracking.</param>
@@ -28,7 +28,7 @@ public interface IPerCallHostResolver
/// <summary>
/// Resolve the host name for the given driver-side full reference. Returned value is
/// used as the <c>hostName</c> argument to the Phase 6.1 <c>CapabilityInvoker</c> so
/// per-host breaker isolation + per-host bulkhead accounting both kick in.
/// per-host breaker isolation + per-host in-flight accounting both kick in.
/// </summary>
/// <param name="fullReference">The full reference of the tag or resource.</param>
/// <returns>The host name responsible for serving the reference.</returns>
@@ -0,0 +1,33 @@
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
/// <summary>
/// Legal ranges for per-capability resilience policy values. Single source of truth shared by
/// <c>DriverResilienceOptionsParser</c> (clamp + diagnostic), <c>DriverResiliencePipelineBuilder</c>
/// (last-resort clamp so a pipeline build can never throw Polly's ValidationException), and the
/// AdminUI <c>ResilienceFormModel</c> (authoring-time validation messages). Derived from
/// Polly.Core 8.6.6's option validation: <c>TimeoutStrategyOptions.Timeout</c> ∈ [10 ms, 24 h];
/// <c>CircuitBreakerStrategyOptions.MinimumThroughput</c> ≥ 2.
/// </summary>
public static class ResiliencePolicyRanges
{
/// <summary>Minimum per-call timeout. Whole-second granularity puts the floor at 1 s (Polly's is 10 ms).</summary>
public const int MinTimeoutSeconds = 1;
/// <summary>Maximum per-call timeout — Polly's TimeoutStrategyOptions ceiling (24 h).</summary>
public const int MaxTimeoutSeconds = 86_400;
/// <summary>Retry floor; 0 = no retry strategy is added to the pipeline.</summary>
public const int MinRetryCount = 0;
/// <summary>Operational retry cap (Polly itself allows int.MaxValue; 100 exponential-backoff attempts ≈ minutes).</summary>
public const int MaxRetryCount = 100;
/// <summary>Breaker floor; 0 = no breaker strategy is added (the Tier C posture).</summary>
public const int MinBreakerFailureThreshold = 0;
/// <summary>Polly's MinimumThroughput floor — an enabled breaker needs a threshold of at least 2.</summary>
public const int MinEnabledBreakerFailureThreshold = 2;
/// <summary>Sanity cap on the breaker threshold (Polly has no ceiling; beyond this it can never trip in the 30 s window).</summary>
public const int MaxBreakerFailureThreshold = 10_000;
}
@@ -9,9 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
/// <remarks>
/// Applied to tag-definition POCOs
/// (e.g. <c>ModbusTagDefinition</c>, <c>S7TagDefinition</c>, OPC UA client tag rows) at the
/// property or record level. The <c>CapabilityInvoker</c> in <c>ZB.MOM.WW.OtOpcUa.Core.Resilience</c>
/// reads this attribute via reflection once at driver-init time and caches the result; no
/// per-write reflection cost.
/// property or record level. <b>Reserved for the per-tag opt-in; NOT yet consulted</b> (R2-02/S-8) —
/// the dispatch site (<c>DriverInstanceActor.HandleWriteAsync</c>) currently treats every write as
/// non-idempotent, so no tag retries a write today. Threading tag-definition metadata to the dispatch
/// seam is the documented forward path; when it ships, the <c>CapabilityInvoker</c> non-idempotent arm
/// and the parser's Write/AlarmAcknowledge no-retry clamp are the two sites that relax.
/// </remarks>
[AttributeUsage(AttributeTargets.Property | AttributeTargets.Class | AttributeTargets.Struct, AllowMultiple = false, Inherited = true)]
public sealed class WriteIdempotentAttribute : Attribute
@@ -52,7 +52,7 @@ public sealed class AlarmSurfaceInvoker
/// <summary>
/// Subscribe to alarm events for a set of source node ids, fanning out by resolved host
/// so per-host breakers / bulkheads apply. Returns one handle per host — callers that
/// so per-host breakers apply. Returns one handle per host — callers that
/// don't care about per-host separation may concatenate them. Each returned handle wraps
/// the driver's opaque handle together with its resolved host so <see cref="UnsubscribeAsync"/>
/// routes through the same host's pipeline that the subscription was created on.
@@ -28,6 +28,8 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
private readonly string _driverType;
private readonly Func<DriverResilienceOptions> _optionsAccessor;
private readonly DriverResilienceStatusTracker? _statusTracker;
private readonly long _optionsGeneration;
private DriverResilienceOptions? _noRetryWriteOptions;
/// <summary>
/// Construct an invoker for one driver instance.
@@ -39,13 +41,19 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
/// pipeline-invalidate can take effect without restarting the invoker.
/// </param>
/// <param name="driverType">Driver type name for structured-log enrichment (e.g. <c>"Modbus"</c>).</param>
/// <param name="statusTracker">Optional resilience-status tracker. When wired, every capability call records start/complete so Admin <c>/hosts</c> can surface <see cref="ResilienceStatusSnapshot.CurrentInFlight"/> as the bulkhead-depth proxy.</param>
/// <param name="statusTracker">Optional resilience-status tracker. When wired, every capability call records start/complete so Admin <c>/hosts</c> can surface <see cref="ResilienceStatusSnapshot.CurrentInFlight"/> as the in-flight-call-depth proxy.</param>
/// <param name="optionsGeneration">
/// Options epoch for this invoker (01/S-7). Threaded into every pipeline-cache lookup so a lingering
/// old child's late re-cache can never poison a newer invoker's pipeline. Defaults to 0 so existing
/// callers/tests are unaffected.
/// </param>
public CapabilityInvoker(
DriverResiliencePipelineBuilder builder,
string driverInstanceId,
Func<DriverResilienceOptions> optionsAccessor,
string driverType = "Unknown",
DriverResilienceStatusTracker? statusTracker = null)
DriverResilienceStatusTracker? statusTracker = null,
long optionsGeneration = 0)
{
ArgumentNullException.ThrowIfNull(builder);
ArgumentNullException.ThrowIfNull(optionsAccessor);
@@ -55,6 +63,7 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
_driverType = driverType;
_optionsAccessor = optionsAccessor;
_statusTracker = statusTracker;
_optionsGeneration = optionsGeneration;
}
/// <inheritdoc />
@@ -116,27 +125,44 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
if (!isIdempotent)
{
// Snapshot the options exactly once per call — invoking _optionsAccessor twice can
// (a) observe two different snapshots if an Admin edit lands between them and
// (b) wastes an allocation on the per-write hot path (Phase 6.1 1% pipeline budget).
var snapshot = _optionsAccessor();
var noRetryOptions = snapshot with
// Build the no-retry options snapshot ONCE per invoker lifetime (01/P-4). The options are
// immutable for the invoker's lifetime — a ResilienceConfig change respawns the driver child,
// which constructs a fresh invoker (#13 respawn-on-change) — so lazily caching the snapshot is
// safe and removes the per-write allocation the previous once-per-call snapshot incurred. The
// null-check-assign tolerates a benign first-call race (both threads build an equivalent record).
var noRetryOptions = _noRetryWriteOptions ??= BuildNoRetryWriteOptions();
var pipeline = _builder.GetOrCreate(_driverInstanceId, $"{hostName}::non-idempotent", DriverCapability.Write, noRetryOptions, _optionsGeneration);
// Record in-flight accounting on the CALLER's host (the "::non-idempotent" suffix is a
// pipeline-cache key, not an operator-facing host) so Admin /hosts sees write depth too.
_statusTracker?.RecordCallStart(_driverInstanceId, hostName);
try
{
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
using (LogContextEnricher.Push(_driverInstanceId, _driverType, DriverCapability.Write, LogContextEnricher.NewCorrelationId()))
{
[DriverCapability.Write] = snapshot.Resolve(DriverCapability.Write) with { RetryCount = 0 },
},
};
var pipeline = _builder.GetOrCreate(_driverInstanceId, $"{hostName}::non-idempotent", DriverCapability.Write, noRetryOptions);
using (LogContextEnricher.Push(_driverInstanceId, _driverType, DriverCapability.Write, LogContextEnricher.NewCorrelationId()))
return await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false);
}
}
finally
{
return await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false);
_statusTracker?.RecordCallComplete(_driverInstanceId, hostName);
}
}
return await ExecuteAsync(DriverCapability.Write, hostName, callSite, cancellationToken).ConfigureAwait(false);
}
private DriverResilienceOptions BuildNoRetryWriteOptions()
{
var snapshot = _optionsAccessor();
return snapshot with
{
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
{
[DriverCapability.Write] = snapshot.Resolve(DriverCapability.Write) with { RetryCount = 0 },
},
};
}
private ResiliencePipeline ResolvePipeline(DriverCapability capability, string hostName) =>
_builder.GetOrCreate(_driverInstanceId, hostName, capability, _optionsAccessor());
_builder.GetOrCreate(_driverInstanceId, hostName, capability, _optionsAccessor(), _optionsGeneration);
}
@@ -19,11 +19,13 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
///
/// <para>Options are snapshotted once per <see cref="Create"/> (a driver actor's options are fixed
/// for its lifetime — a ResilienceConfig change respawns the child), so the invoker's per-call
/// options accessor is allocation-free on the hot path. Because the pipeline builder caches
/// pipelines keyed on <c>(instance, host, capability)</c> and <b>ignores options on a cache hit</b>,
/// <see cref="Create"/> first <see cref="DriverResiliencePipelineBuilder.Invalidate"/>s any pipelines
/// cached for the instance so a respawn with changed options rebuilds them (rather than silently
/// reusing the stale pipeline). On a first spawn this removes nothing.</para>
/// options accessor is allocation-free on the hot path. The pipeline builder caches pipelines keyed on
/// <c>(instance, host, capability, generation)</c> and ignores options on a cache hit; each
/// <see cref="Create"/> stamps a fresh monotonic <b>generation</b> (01/S-7) so the new invoker can only
/// read pipelines it built — a lingering old child's post-invalidate re-cache lands under its own older
/// generation and is unreachable. <see cref="Create"/> still
/// <see cref="DriverResiliencePipelineBuilder.Invalidate"/>s the instance's pipelines as cleanup (bounds
/// the cache), but correctness now rests on the generation key, not the invalidate timing.</para>
/// </remarks>
public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFactory
{
@@ -31,6 +33,7 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
private readonly DriverResilienceStatusTracker _statusTracker;
private readonly Func<string, DriverTier> _tierResolver;
private readonly ILogger? _logger;
private long _generation;
/// <summary>Construct the factory over the shared resilience infrastructure.</summary>
/// <param name="builder">Process-singleton Polly pipeline builder (shared pipeline cache).</param>
@@ -65,9 +68,14 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
driverInstanceId, driverType, diagnostic);
}
// Drop any pipelines cached for this instance under the PREVIOUS options — the builder keys on
// (instance, host, capability) and reuses a cached pipeline regardless of options, so a respawn
// with a changed ResilienceConfig would otherwise keep serving the stale pipeline. No-op on first spawn.
// Stamp a fresh options generation for this invoker. The generation is part of the pipeline-cache
// key (01/S-7), so this invoker can only ever read pipelines it built — a lingering old child's
// late re-cache lands under its OWN (older) generation and is unreachable here. Correctness rests on
// the epoch key, not on the timing of the invalidate below.
var generation = Interlocked.Increment(ref _generation);
// Cleanup (no longer the correctness mechanism): drop this instance's cached pipelines across all
// generations so lingering old-generation entries don't accumulate. No-op on first spawn.
_builder.Invalidate(driverInstanceId);
return new CapabilityInvoker(
@@ -75,6 +83,7 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
driverInstanceId,
optionsAccessor: () => options,
driverType: driverType,
statusTracker: _statusTracker);
statusTracker: _statusTracker,
optionsGeneration: generation);
}
}
@@ -19,15 +19,6 @@ public sealed record DriverResilienceOptions
public IReadOnlyDictionary<DriverCapability, CapabilityPolicy> CapabilityPolicies { get; init; }
= new Dictionary<DriverCapability, CapabilityPolicy>();
/// <summary>Bulkhead (max concurrent in-flight calls) for every capability. Default 32.</summary>
public int BulkheadMaxConcurrent { get; init; } = 32;
/// <summary>
/// Bulkhead queue depth. Zero = no queueing; overflow fails fast with
/// <c>BulkheadRejectedException</c>. Default 64.
/// </summary>
public int BulkheadMaxQueue { get; init; } = 64;
/// <summary>
/// Periodic scheduled recycle interval for Tier C out-of-process hosts, in seconds.
/// Null (the default) = no scheduled recycle; the driver's Host process keeps running
@@ -69,8 +60,10 @@ public sealed record DriverResilienceOptions
/// <summary>
/// Per-tier per-capability default policy table, per the Phase 6.1
/// Stream A.2 specification. Retries skipped on <see cref="DriverCapability.Write"/> and
/// <see cref="DriverCapability.AlarmAcknowledge"/> regardless of tier.
/// Stream A.2 specification. The parser enforces no-retry on
/// <see cref="DriverCapability.Write"/> and <see cref="DriverCapability.AlarmAcknowledge"/>
/// as an invariant (R2-02/S-8) — a JSON retryCount override on those capabilities is forced back
/// to 0; per-tag opt-in via <see cref="WriteIdempotentAttribute"/> is the future relaxation.
/// </summary>
/// <param name="tier">The driver tier to get defaults for.</param>
/// <returns>The default policy dictionary for the specified tier.</returns>
@@ -13,8 +13,6 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
/// <para>Example JSON shape per Phase 6.1 Stream A.2:</para>
/// <code>
/// {
/// "bulkheadMaxConcurrent": 16,
/// "bulkheadMaxQueue": 64,
/// "capabilityPolicies": {
/// "Read": { "timeoutSeconds": 5, "retryCount": 5, "breakerFailureThreshold": 3 },
/// "Write": { "timeoutSeconds": 5, "retryCount": 0, "breakerFailureThreshold": 5 }
@@ -83,15 +81,16 @@ public static class DriverResilienceOptionsParser
{
if (!Enum.TryParse<DriverCapability>(capName, ignoreCase: true, out var capability))
{
parseDiagnostic ??= $"Unknown capability '{capName}' in ResilienceConfig; skipped.";
AppendDiagnostic(ref parseDiagnostic, $"Unknown capability '{capName}' in ResilienceConfig; skipped.");
continue;
}
var basePolicy = merged[capability];
merged[capability] = new CapabilityPolicy(
var mergedPolicy = new CapabilityPolicy(
TimeoutSeconds: overridePolicy.TimeoutSeconds ?? basePolicy.TimeoutSeconds,
RetryCount: overridePolicy.RetryCount ?? basePolicy.RetryCount,
BreakerFailureThreshold: overridePolicy.BreakerFailureThreshold ?? basePolicy.BreakerFailureThreshold);
merged[capability] = ClampPolicy(capability, mergedPolicy, basePolicy, ref parseDiagnostic);
}
}
@@ -102,9 +101,9 @@ public static class DriverResilienceOptionsParser
if (shape.RecycleIntervalSeconds is int secs)
{
if (secs <= 0)
parseDiagnostic ??= $"RecycleIntervalSeconds must be positive; got {secs} — ignoring.";
AppendDiagnostic(ref parseDiagnostic, $"RecycleIntervalSeconds must be positive; got {secs} — ignoring.");
else if (tier != DriverTier.C)
parseDiagnostic ??= $"RecycleIntervalSeconds is Tier C only; Tier {tier} driver will not scheduled-recycle.";
AppendDiagnostic(ref parseDiagnostic, $"RecycleIntervalSeconds is Tier C only; Tier {tier} driver will not scheduled-recycle.");
else
recycleIntervalSeconds = secs;
}
@@ -113,18 +112,90 @@ public static class DriverResilienceOptionsParser
{
Tier = tier,
CapabilityPolicies = merged,
BulkheadMaxConcurrent = shape.BulkheadMaxConcurrent ?? baseOptions.BulkheadMaxConcurrent,
BulkheadMaxQueue = shape.BulkheadMaxQueue ?? baseOptions.BulkheadMaxQueue,
RecycleIntervalSeconds = recycleIntervalSeconds,
};
}
/// <summary>
/// Clamp a merged per-capability override to the legal <see cref="ResiliencePolicyRanges"/> so a
/// downstream Polly pipeline build can never throw <c>ValidationException</c> (01/S-6). Each clamp
/// appends a diagnostic naming the field + what happened. A non-positive timeout snaps to the
/// capability's <paramref name="tierDefault"/> (a nonsense value, not merely an extreme — snapping
/// to 1 s could be a 30× surprise on a Discover policy); everything else clamps to the nearest legal
/// value. Preserves maximum operator intent while never bricking.
/// </summary>
private static CapabilityPolicy ClampPolicy(
DriverCapability capability,
CapabilityPolicy merged,
CapabilityPolicy tierDefault,
ref string? diagnostic)
{
var timeout = merged.TimeoutSeconds;
if (timeout <= 0)
{
AppendDiagnostic(ref diagnostic,
$"{capability}.timeoutSeconds {timeout} is non-positive; using the tier default ({tierDefault.TimeoutSeconds}s).");
timeout = tierDefault.TimeoutSeconds;
}
else if (timeout > ResiliencePolicyRanges.MaxTimeoutSeconds)
{
AppendDiagnostic(ref diagnostic,
$"{capability}.timeoutSeconds {timeout} exceeds the max ({ResiliencePolicyRanges.MaxTimeoutSeconds}s); clamped.");
timeout = ResiliencePolicyRanges.MaxTimeoutSeconds;
}
var retry = merged.RetryCount;
if (retry < ResiliencePolicyRanges.MinRetryCount)
{
AppendDiagnostic(ref diagnostic, $"{capability}.retryCount {retry} is negative; clamped to {ResiliencePolicyRanges.MinRetryCount}.");
retry = ResiliencePolicyRanges.MinRetryCount;
}
else if (retry > ResiliencePolicyRanges.MaxRetryCount)
{
AppendDiagnostic(ref diagnostic, $"{capability}.retryCount {retry} exceeds the cap ({ResiliencePolicyRanges.MaxRetryCount}); clamped.");
retry = ResiliencePolicyRanges.MaxRetryCount;
}
// Spec invariant (01/S-8 = 03/S12): Write and AlarmAcknowledge are write-shaped — a duplicate is
// not harmless (pulse coils, counter increments, Galaxy supervisory writes, alarm acks). Retries are
// "skipped regardless of tier" per Phase 6.1; the tier defaults are already 0, so only an override can
// differ — force it back to 0 here so the JSON→options funnel enforces the invariant. When per-tag
// opt-in via WriteIdempotentAttribute ships, this is the one line to relax.
if ((capability == DriverCapability.Write || capability == DriverCapability.AlarmAcknowledge) && retry > 0)
{
AppendDiagnostic(ref diagnostic,
$"{capability} never retries (writes are treated as non-idempotent until per-tag opt-in ships); retryCount override ignored.");
retry = 0;
}
var breaker = merged.BreakerFailureThreshold;
if (breaker < ResiliencePolicyRanges.MinBreakerFailureThreshold)
{
AppendDiagnostic(ref diagnostic, $"{capability}.breakerFailureThreshold {breaker} is negative; breaker disabled.");
breaker = ResiliencePolicyRanges.MinBreakerFailureThreshold;
}
else if (breaker == 1)
{
AppendDiagnostic(ref diagnostic,
$"{capability}.breakerFailureThreshold 1 is below Polly's minimum throughput; raised to {ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold} (closest to open-on-first-failure intent).");
breaker = ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold;
}
else if (breaker > ResiliencePolicyRanges.MaxBreakerFailureThreshold)
{
AppendDiagnostic(ref diagnostic,
$"{capability}.breakerFailureThreshold {breaker} exceeds the cap ({ResiliencePolicyRanges.MaxBreakerFailureThreshold}); clamped.");
breaker = ResiliencePolicyRanges.MaxBreakerFailureThreshold;
}
return merged with { TimeoutSeconds = timeout, RetryCount = retry, BreakerFailureThreshold = breaker };
}
/// <summary>Append a diagnostic to the running channel (joined with "; ") so multiple clamps all surface.</summary>
private static void AppendDiagnostic(ref string? diagnostic, string message)
=> diagnostic = diagnostic is null ? message : $"{diagnostic}; {message}";
private sealed class ResilienceConfigShape
{
/// <summary>Gets or sets the maximum concurrent bulkhead requests.</summary>
public int? BulkheadMaxConcurrent { get; set; }
/// <summary>Gets or sets the maximum bulkhead queue size.</summary>
public int? BulkheadMaxQueue { get; set; }
/// <summary>Gets or sets the scheduled recycle interval in seconds.</summary>
public int? RecycleIntervalSeconds { get; set; }
/// <summary>Gets or sets the per-capability resilience policies.</summary>
@@ -15,11 +15,17 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
/// </summary>
/// <remarks>
/// Per-device isolation. Composition from outside-in:
/// <b>Timeout → Retry (when capability permits) → Circuit Breaker (when tier permits) → Bulkhead</b>.
/// <b>Timeout → Retry (when capability permits) → Circuit Breaker (when tier permits)</b>.
///
/// <para>Pipeline resolution is lock-free on the hot path: the inner
/// <see cref="ConcurrentDictionary{TKey,TValue}"/> caches a <see cref="ResiliencePipeline"/> per key;
/// first-call cost is one <see cref="ResiliencePipelineBuilder"/>.Build. Thereafter reads are O(1).</para>
///
/// <para><b>Never-throws invariant (01/S-6):</b> <see cref="Build"/> clamps every policy value to
/// <see cref="ResiliencePolicyRanges"/> before constructing strategy options, so a hostile
/// <see cref="DriverResilienceOptions"/> (constructed directly, bypassing the parser) can never make
/// the memoizing <c>GetOrAdd</c> factory throw Polly's <c>ValidationException</c> — which would
/// otherwise brick every capability call for that key.</para>
/// </remarks>
public sealed class DriverResiliencePipelineBuilder
{
@@ -33,7 +39,7 @@ public sealed class DriverResiliencePipelineBuilder
/// <param name="statusTracker">When non-null, every built pipeline wires Polly telemetry into
/// the tracker — retries increment <c>ConsecutiveFailures</c>, breaker-open stamps
/// <c>LastBreakerOpenUtc</c>, breaker-close resets failures. Feeds Admin <c>/hosts</c> +
/// the Polly bulkhead-depth column. Absent tracker means no telemetry (unit tests +
/// the in-flight-call-depth column. Absent tracker means no telemetry (unit tests +
/// deployments that don't care about resilience observability).</param>
/// <param name="logger">When non-null, retry / circuit-breaker-open / breaker-close events are
/// logged (instance + host + capability + attempt/exception). This is the operator-facing
@@ -63,23 +69,35 @@ public sealed class DriverResiliencePipelineBuilder
/// </param>
/// <param name="capability">Which capability surface is being called.</param>
/// <param name="options">Per-driver-instance options (tier + per-capability overrides).</param>
/// <param name="optionsGeneration">
/// Monotonic options epoch stamped by <see cref="DriverCapabilityInvokerFactory"/> per invoker
/// (01/S-7). Part of the cache key so a lingering old child's post-invalidate re-cache lands under
/// its OWN (old) generation — a key the new invoker's generation never reads. Defaults to 0 so
/// every existing caller/test compiles unchanged.
/// </param>
/// <returns>The cached or newly built resilience pipeline for the key.</returns>
public ResiliencePipeline GetOrCreate(
string driverInstanceId,
string hostName,
DriverCapability capability,
DriverResilienceOptions options)
DriverResilienceOptions options,
long optionsGeneration = 0)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentException.ThrowIfNullOrWhiteSpace(hostName);
var key = new PipelineKey(driverInstanceId, hostName, capability);
var key = new PipelineKey(driverInstanceId, hostName, capability, optionsGeneration);
return _pipelines.GetOrAdd(key, static (k, state) => Build(
k.DriverInstanceId, k.HostName, state.capability, state.options, state.timeProvider, state.tracker, state.logger),
(capability, options, timeProvider: _timeProvider, tracker: _statusTracker, logger: _logger));
}
/// <summary>Drop cached pipelines for one driver instance (e.g. on ResilienceConfig change). Test + Admin-reload use.</summary>
/// <summary>
/// Drop cached pipelines for one driver instance (e.g. on ResilienceConfig change), across ALL
/// option generations. With generation-keyed entries (01/S-7) correctness no longer depends on this
/// sweep — a stale-epoch re-cache is already unreachable by the new generation — but it bounds the
/// cache by clearing lingering old-generation entries on the next respawn. Test + Admin-reload use.
/// </summary>
/// <param name="driverInstanceId">The driver instance ID whose pipelines should be invalidated.</param>
/// <returns>The number of pipelines removed.</returns>
public int Invalidate(string driverInstanceId)
@@ -108,16 +126,23 @@ public sealed class DriverResiliencePipelineBuilder
var policy = options.Resolve(capability);
var builder = new ResiliencePipelineBuilder { TimeProvider = timeProvider };
// Belt-and-braces: clamp the resolved policy to ResiliencePolicyRanges so building a pipeline can
// NEVER throw Polly's ValidationException (01/S-6). The parser already clamps every JSON→options
// funnel, but DriverResilienceOptions is a public record constructible by paths that never traverse
// the parser (tests today; any future caller), so this makes "a pipeline build never throws" a
// builder invariant. Three integer ops once per pipeline build (not per call).
var timeoutSeconds = Math.Clamp(
policy.TimeoutSeconds, ResiliencePolicyRanges.MinTimeoutSeconds, ResiliencePolicyRanges.MaxTimeoutSeconds);
builder.AddTimeout(new TimeoutStrategyOptions
{
Timeout = TimeSpan.FromSeconds(policy.TimeoutSeconds),
Timeout = TimeSpan.FromSeconds(timeoutSeconds),
});
if (policy.RetryCount > 0)
{
var retryOptions = new RetryStrategyOptions
{
MaxRetryAttempts = policy.RetryCount,
MaxRetryAttempts = Math.Min(policy.RetryCount, ResiliencePolicyRanges.MaxRetryCount),
BackoffType = DelayBackoffType.Exponential,
UseJitter = true,
Delay = TimeSpan.FromMilliseconds(100),
@@ -143,7 +168,7 @@ public sealed class DriverResiliencePipelineBuilder
var breakerOptions = new CircuitBreakerStrategyOptions
{
FailureRatio = 1.0,
MinimumThroughput = policy.BreakerFailureThreshold,
MinimumThroughput = Math.Max(ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold, policy.BreakerFailureThreshold),
SamplingDuration = TimeSpan.FromSeconds(30),
BreakDuration = TimeSpan.FromSeconds(15),
ShouldHandle = new PredicateBuilder().Handle<Exception>(ex => ex is not OperationCanceledException),
@@ -175,5 +200,5 @@ public sealed class DriverResiliencePipelineBuilder
return builder.Build();
}
private readonly record struct PipelineKey(string DriverInstanceId, string HostName, DriverCapability Capability);
private readonly record struct PipelineKey(string DriverInstanceId, string HostName, DriverCapability Capability, long Generation);
}
@@ -101,7 +101,7 @@ public sealed class DriverResilienceStatusTracker
/// <summary>
/// Record the entry of a capability call for this (instance, host). Increments the
/// in-flight counter used as the <see cref="ResilienceStatusSnapshot.CurrentInFlight"/>
/// surface (a cheap stand-in for Polly bulkhead depth). Paired with
/// surface (in-flight capability-call depth). Paired with
/// <see cref="RecordCallComplete"/>; callers use try/finally.
/// </summary>
/// <param name="driverInstanceId">The driver instance identifier.</param>
@@ -159,7 +159,7 @@ public sealed record ResilienceStatusSnapshot
/// <summary>
/// In-flight capability calls against this (instance, host). Bumped on call entry +
/// decremented on completion. Feeds <c>DriverInstanceResilienceStatus.CurrentBulkheadDepth</c>
/// for Admin <c>/hosts</c> — a cheap proxy for the Polly bulkhead depth until the full
/// for Admin <c>/hosts</c> — in-flight capability-call depth until the full
/// telemetry observer lands.
/// </summary>
public int CurrentInFlight { get; init; }
@@ -14,7 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip;
/// <para>Wire layer is libplctag 1.6.x. Per-device host addresses use
/// the <c>ab://gateway[:port]/cip-path</c> canonical form parsed via
/// <see cref="AbCipHostAddress.TryParse"/>; those strings become the <c>hostName</c> key
/// for Polly bulkhead + circuit-breaker isolation.</para>
/// for Polly per-host circuit-breaker isolation.</para>
///
/// <para>Tier A — in-process, shares server lifetime, no
/// sidecar. <see cref="ReinitializeAsync"/> is the Tier-B escape hatch for recovering
@@ -3,7 +3,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip;
/// <summary>
/// Parsed <c>ab://gateway[:port]/cip-path</c> host-address string used by the AbCip driver
/// as the <c>hostName</c> key across <see cref="Core.Abstractions.IHostConnectivityProbe"/>,
/// <see cref="Core.Abstractions.IPerCallHostResolver"/>, and the Polly bulkhead key
/// <see cref="Core.Abstractions.IPerCallHostResolver"/>, and the Polly per-host pipeline key
/// <c>(DriverInstanceId, hostName)</c>.
/// </summary>
/// <remarks>
@@ -4,7 +4,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
/// <summary>
/// FOCAS driver configuration. One instance supports N CNC devices. Each device gets its own
/// <c>(DriverInstanceId, HostAddress)</c> bulkhead key at the Phase 6.1 resilience layer.
/// <c>(DriverInstanceId, HostAddress)</c> per-host pipeline key at the Phase 6.1 resilience layer.
/// </summary>
public sealed class FocasDriverOptions
{
@@ -6,13 +6,23 @@
<p class="form-text mb-3">Blank fields use the driver type's stability-tier defaults
(see <span class="mono">docs/v2/driver-stability.md</span>). Set only what you need to override.</p>
@if (_m.ParseFailed)
{
<div class="alert alert-danger" role="alert">
<strong>Stored resilience config could not be parsed.</strong>
Editing is locked so an accidental keystroke can't overwrite it. Fix the stored JSON directly,
or discard it and start from tier defaults.
<div class="mt-2">
<button type="button" class="btn btn-sm btn-outline-danger" @onclick="DiscardAsync">
Discard stored JSON and start blank
</button>
</div>
</div>
}
<div class="row g-3">
<div class="col-md-4"><label class="form-label">Bulkhead max concurrent</label>
<input type="number" class="form-control form-control-sm" @bind="_m.BulkheadMaxConcurrent" @bind:after="EmitAsync" placeholder="tier default" /></div>
<div class="col-md-4"><label class="form-label">Bulkhead max queue</label>
<input type="number" class="form-control form-control-sm" @bind="_m.BulkheadMaxQueue" @bind:after="EmitAsync" placeholder="tier default" /></div>
<div class="col-md-4"><label class="form-label">Recycle interval (s, Tier C only)</label>
<input type="number" class="form-control form-control-sm" @bind="_m.RecycleIntervalSeconds" @bind:after="EmitAsync" placeholder="none" /></div>
<input type="number" class="form-control form-control-sm" @bind="_m.RecycleIntervalSeconds" @bind:after="EmitAsync" placeholder="none" disabled="@_m.ParseFailed" /></div>
</div>
<div class="table-wrap mt-3">
@@ -22,20 +32,37 @@
@foreach (var cap in ResilienceFormModel.Capabilities)
{
var row = _m.Policies[cap];
var writeShaped = cap is "Write" or "AlarmAcknowledge";
<tr>
<td class="mono">@cap</td>
<td><input type="number" class="form-control form-control-sm" @bind="row.TimeoutSeconds" @bind:after="EmitAsync" placeholder="default" /></td>
<td><input type="number" class="form-control form-control-sm" @bind="row.RetryCount" @bind:after="EmitAsync" placeholder="default" /></td>
<td><input type="number" class="form-control form-control-sm" @bind="row.BreakerFailureThreshold" @bind:after="EmitAsync" placeholder="default" /></td>
<td><input type="number" class="form-control form-control-sm" @bind="row.TimeoutSeconds" @bind:after="EmitAsync" placeholder="default" disabled="@_m.ParseFailed" /></td>
<td><input type="number" class="form-control form-control-sm" @bind="row.RetryCount" @bind:after="EmitAsync" placeholder="@(writeShaped ? "never" : "default")" disabled="@(writeShaped || _m.ParseFailed)" title="@(writeShaped ? "Writes/acks never retry — per-tag opt-in not yet available" : null)" /></td>
<td><input type="number" class="form-control form-control-sm" @bind="row.BreakerFailureThreshold" @bind:after="EmitAsync" placeholder="default" disabled="@_m.ParseFailed" /></td>
</tr>
}
</tbody>
</table>
</div>
@{ var _warnings = _m.Validate(); }
@if (_warnings.Count > 0)
{
<div class="alert alert-warning mt-3" role="alert">
<strong>Out-of-range values will be clamped to a safe value on deploy:</strong>
<ul class="mb-0">
@foreach (var w in _warnings)
{
<li>@w</li>
}
</ul>
</div>
}
<details class="mt-3">
<summary class="small text-muted">Raw JSON (advanced)</summary>
<pre class="form-control form-control-sm mono mt-2" style="white-space:pre-wrap;min-height:3rem;">@(_m.ToJson() ?? "(null — all tier defaults)")</pre>
@* Render the bound STORED value, not the re-serialized model — shows the truth in both the
healthy and the malformed case. *@
<pre class="form-control form-control-sm mono mt-2" style="white-space:pre-wrap;min-height:3rem;">@(ResilienceConfig ?? "(null — all tier defaults)")</pre>
</details>
</div>
</section>
@@ -59,9 +86,22 @@
private async Task EmitAsync()
{
// Never emit while the stored config is unparseable — the section is locked, and an unrelated
// keystroke must never silently overwrite what we couldn't read (04/C-7).
if (_m.ParseFailed) return;
var json = _m.ToJson();
_lastParsed = json;
ResilienceConfig = json;
await ResilienceConfigChanged.InvokeAsync(json);
}
private async Task DiscardAsync()
{
// Explicit, visible destruction — replace the unparseable blob with a blank (tier-default) config.
_m = new ResilienceFormModel();
_lastParsed = null;
ResilienceConfig = null;
await ResilienceConfigChanged.InvokeAsync(null);
}
}
@@ -1,23 +1,28 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Nodes;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers;
/// <summary>
/// Mutable, all-nullable form model for the driver resilience override. Binds the typed
/// fields in DriverResilienceSection; null/blank = "use the driver's tier default", so a
/// blank form serializes back to null (preserving DriverInstance.ResilienceConfig = null).
/// Emits / reads the exact override JSON shape DriverResilienceOptionsParser consumes.
/// Mutable, all-nullable form model for the driver resilience override. Binds the typed fields in
/// DriverResilienceSection; null/blank = "use the driver's tier default", so a blank form serializes
/// back to null (preserving DriverInstance.ResilienceConfig = null). Emits / reads the exact override
/// JSON shape DriverResilienceOptionsParser consumes.
/// <para>
/// Round-trip is <b>non-lossy</b> (04/C-7): the stored JSON is kept as a <see cref="JsonObject"/> bag,
/// and <see cref="ToJson"/> overlays only the known fields onto that bag — so unknown top-level keys,
/// unknown capability entries, and unknown per-policy fields all survive a load→save, mirroring the
/// driver tag editors' preserve-unknown discipline (<c>TagConfigJson</c>). A stored blob that could not
/// be parsed sets <see cref="ParseFailed"/> and is <b>never</b> overwritten — <see cref="ToJson"/>
/// returns the original text verbatim.
/// </para>
/// </summary>
public sealed class ResilienceFormModel
{
public static readonly string[] Capabilities =
["Read", "Write", "Discover", "Subscribe", "Probe", "AlarmSubscribe", "AlarmAcknowledge", "HistoryRead"];
/// <summary>Gets or sets the bulkhead max-concurrency override; null = use the tier default.</summary>
public int? BulkheadMaxConcurrent { get; set; }
/// <summary>Gets or sets the bulkhead max-queue-length override; null = use the tier default.</summary>
public int? BulkheadMaxQueue { get; set; }
/// <summary>Gets or sets the driver recycle-interval override, in seconds; null = use the tier default.</summary>
public int? RecycleIntervalSeconds { get; set; }
@@ -26,6 +31,18 @@ public sealed class ResilienceFormModel
public Dictionary<string, CapabilityRow> Policies { get; set; } =
Capabilities.ToDictionary(c => c, _ => new CapabilityRow(), StringComparer.OrdinalIgnoreCase);
/// <summary>
/// True when the stored ResilienceConfig could not be parsed as JSON. The section locks editing and
/// <see cref="ToJson"/> returns <see cref="RawStoredJson"/> unchanged so an unrelated keystroke can
/// never silently overwrite an unreadable column.
/// </summary>
public bool ParseFailed { get; private set; }
/// <summary>The original stored text when <see cref="ParseFailed"/> — returned verbatim by <see cref="ToJson"/>.</summary>
public string? RawStoredJson { get; private set; }
private JsonObject _bag = new();
/// <summary>Per-capability timeout/retry/breaker override row; null fields fall back to the tier default.</summary>
public sealed class CapabilityRow
{
@@ -39,88 +56,157 @@ public sealed class ResilienceFormModel
public bool IsEmpty => TimeoutSeconds is null && RetryCount is null && BreakerFailureThreshold is null;
}
private static readonly JsonSerializerOptions ReadOpts = new() { PropertyNameCaseInsensitive = true };
private static readonly JsonSerializerOptions WriteOpts = new()
/// <summary>
/// Range-validate the per-capability overrides against <see cref="ResiliencePolicyRanges"/> and return
/// one human-readable message per violation (naming capability + field + legal range). This is
/// authoring-time <b>feedback</b>, not the enforcement layer — the parser's clamp is the authoritative
/// guard, so the form still emits. Mirrors the driver tag editors' <c>Validate()</c> idiom (01/S-6).
/// </summary>
/// <returns>A list of range-violation messages; empty when every override is in range or blank.</returns>
public IReadOnlyList<string> Validate()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
var msgs = new List<string>();
foreach (var (cap, row) in Policies)
{
if (row.TimeoutSeconds is { } t &&
(t < ResiliencePolicyRanges.MinTimeoutSeconds || t > ResiliencePolicyRanges.MaxTimeoutSeconds))
msgs.Add($"{cap} timeout must be between {ResiliencePolicyRanges.MinTimeoutSeconds} and " +
$"{ResiliencePolicyRanges.MaxTimeoutSeconds} seconds (got {t}).");
/// <summary>Parses the override JSON into a form model; malformed or blank JSON yields an empty (all-default) form.</summary>
if (row.RetryCount is { } r &&
(r < ResiliencePolicyRanges.MinRetryCount || r > ResiliencePolicyRanges.MaxRetryCount))
msgs.Add($"{cap} retries must be between {ResiliencePolicyRanges.MinRetryCount} and " +
$"{ResiliencePolicyRanges.MaxRetryCount} (got {r}).");
if (row.BreakerFailureThreshold is { } b &&
(b < ResiliencePolicyRanges.MinBreakerFailureThreshold || b == 1 ||
b > ResiliencePolicyRanges.MaxBreakerFailureThreshold))
msgs.Add($"{cap} breaker threshold must be 0 (off) or between " +
$"{ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold} and " +
$"{ResiliencePolicyRanges.MaxBreakerFailureThreshold} (got {b}).");
}
return msgs;
}
/// <summary>
/// Parse the override JSON into a form model, preserving every key in an internal bag. Malformed JSON
/// sets <see cref="ParseFailed"/> + <see cref="RawStoredJson"/> and yields an otherwise-empty model.
/// </summary>
/// <param name="json">The raw resilience-override JSON, or null/blank if there is no override.</param>
/// <returns>A populated form model, or an all-default one when <paramref name="json"/> is blank or malformed.</returns>
/// <returns>A populated form model.</returns>
public static ResilienceFormModel FromJson(string? json)
{
var model = new ResilienceFormModel();
if (string.IsNullOrWhiteSpace(json)) return model;
Shape? shape;
try { shape = JsonSerializer.Deserialize<Shape>(json, ReadOpts); }
catch (JsonException) { return model; } // malformed -> empty form; raw view (next task) shows the text
if (shape is null) return model;
JsonObject bag;
try
{
bag = JsonNode.Parse(json) as JsonObject ?? new JsonObject();
}
catch (JsonException)
{
model.ParseFailed = true;
model.RawStoredJson = json;
return model;
}
model.BulkheadMaxConcurrent = shape.BulkheadMaxConcurrent;
model.BulkheadMaxQueue = shape.BulkheadMaxQueue;
model.RecycleIntervalSeconds = shape.RecycleIntervalSeconds;
if (shape.CapabilityPolicies is not null)
foreach (var (cap, p) in shape.CapabilityPolicies)
if (model.Policies.TryGetValue(cap, out var row))
model._bag = bag;
model.RecycleIntervalSeconds = GetIntOrNull(bag, "recycleIntervalSeconds");
if (bag.TryGetPropertyValue("capabilityPolicies", out var cpNode) && cpNode is JsonObject cp)
{
foreach (var (capName, polNode) in cp)
{
// Known capability names populate rows (reading only the known per-policy fields — unknown
// per-policy fields stay untouched in the bag). Unknown capability entries stay in the bag.
if (polNode is JsonObject pol && model.Policies.TryGetValue(capName, out var row))
{
row.TimeoutSeconds = p.TimeoutSeconds;
row.RetryCount = p.RetryCount;
row.BreakerFailureThreshold = p.BreakerFailureThreshold;
row.TimeoutSeconds = GetIntOrNull(pol, "timeoutSeconds");
row.RetryCount = GetIntOrNull(pol, "retryCount");
row.BreakerFailureThreshold = GetIntOrNull(pol, "breakerFailureThreshold");
}
}
}
return model;
}
/// <summary>Emit only the non-null overrides; returns null when nothing is overridden.</summary>
/// <returns>The serialized override JSON, or null when no field in this form is overridden.</returns>
/// <summary>
/// Overlay the model's non-null known values onto the preserved bag and serialize. Returns null when
/// the result is empty (blank form = tier defaults). When <see cref="ParseFailed"/>, returns the
/// original unparseable text unchanged.
/// </summary>
/// <returns>The serialized override JSON, the original text when parse failed, or null when nothing is set.</returns>
public string? ToJson()
{
var caps = Policies
.Where(kv => !kv.Value.IsEmpty)
.ToDictionary(kv => kv.Key, kv => new PolicyShape
{
TimeoutSeconds = kv.Value.TimeoutSeconds,
RetryCount = kv.Value.RetryCount,
BreakerFailureThreshold = kv.Value.BreakerFailureThreshold,
});
if (ParseFailed) return RawStoredJson;
var hasAny = BulkheadMaxConcurrent is not null || BulkheadMaxQueue is not null
|| RecycleIntervalSeconds is not null || caps.Count > 0;
if (!hasAny) return null;
SetOrRemoveInt(_bag, "recycleIntervalSeconds", RecycleIntervalSeconds);
var shape = new Shape
var cp = _bag.TryGetPropertyValue("capabilityPolicies", out var cpNode) && cpNode is JsonObject existing
? existing
: null;
foreach (var (cap, row) in Policies)
{
BulkheadMaxConcurrent = BulkheadMaxConcurrent,
BulkheadMaxQueue = BulkheadMaxQueue,
RecycleIntervalSeconds = RecycleIntervalSeconds,
CapabilityPolicies = caps.Count > 0 ? caps : null,
};
return JsonSerializer.Serialize(shape, WriteOpts);
// Find this known capability's existing policy object (case-insensitive), else its canonical key.
var polKey = cap;
JsonObject? pol = null;
if (cp is not null)
{
foreach (var (k, v) in cp)
{
if (string.Equals(k, cap, StringComparison.OrdinalIgnoreCase) && v is JsonObject vo)
{
polKey = k;
pol = vo;
break;
}
}
}
if (row.IsEmpty)
{
// Clear the known fields; drop the policy entry only if no unknown fields remain.
if (pol is not null)
{
pol.Remove("timeoutSeconds");
pol.Remove("retryCount");
pol.Remove("breakerFailureThreshold");
if (pol.Count == 0) cp!.Remove(polKey);
}
}
else
{
if (cp is null)
{
cp = new JsonObject();
_bag["capabilityPolicies"] = cp;
}
if (pol is null)
{
pol = new JsonObject();
cp[polKey] = pol;
}
SetOrRemoveInt(pol, "timeoutSeconds", row.TimeoutSeconds);
SetOrRemoveInt(pol, "retryCount", row.RetryCount);
SetOrRemoveInt(pol, "breakerFailureThreshold", row.BreakerFailureThreshold);
}
}
if (cp is not null && cp.Count == 0) _bag.Remove("capabilityPolicies");
return _bag.Count == 0 ? null : _bag.ToJsonString();
}
/// <summary>Wire shape of the resilience-override JSON, as consumed by DriverResilienceOptionsParser.</summary>
private sealed class Shape
{
/// <summary>Gets or sets the bulkhead max-concurrency override.</summary>
public int? BulkheadMaxConcurrent { get; set; }
/// <summary>Gets or sets the bulkhead max-queue-length override.</summary>
public int? BulkheadMaxQueue { get; set; }
/// <summary>Gets or sets the driver recycle-interval override, in seconds.</summary>
public int? RecycleIntervalSeconds { get; set; }
/// <summary>Gets or sets the per-capability overrides, keyed by capability name.</summary>
public Dictionary<string, PolicyShape>? CapabilityPolicies { get; set; }
}
private static int? GetIntOrNull(JsonObject o, string name)
=> o.TryGetPropertyValue(name, out var n) && n is JsonValue v && v.TryGetValue<int>(out var i)
? i
: null;
/// <summary>Wire shape of a single capability's timeout/retry/breaker override.</summary>
private sealed class PolicyShape
private static void SetOrRemoveInt(JsonObject o, string name, int? value)
{
/// <summary>Gets or sets the timeout override, in seconds.</summary>
public int? TimeoutSeconds { get; set; }
/// <summary>Gets or sets the retry-count override.</summary>
public int? RetryCount { get; set; }
/// <summary>Gets or sets the circuit-breaker failure-threshold override.</summary>
public int? BreakerFailureThreshold { get; set; }
if (value is null) o.Remove(name);
else o[name] = JsonValue.Create(value.Value);
}
}
@@ -1627,7 +1627,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
else
{
// Attach the per-instance Phase 6.1 resilience invoker so this driver's capability calls
// route through the retry/breaker/bulkhead/telemetry pipeline. The tier defaults are layered
// route through the retry/breaker/telemetry pipeline. The tier defaults are layered
// with the per-instance ResilienceConfig from the artifact; the pass-through factory yields a
// no-op invoker on nodes without the real resilience factory bound.
var invoker = _invokerFactory.Create(spec.DriverInstanceId, spec.DriverType, spec.ResilienceConfig);
@@ -139,7 +139,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
private readonly IDriver _driver;
/// <summary>Phase 6.1 resilience seam: every guarded driver-capability call this actor makes is
/// routed through this invoker (retry / breaker / bulkhead / tracker telemetry). Defaults to
/// routed through this invoker (retry / breaker / in-flight accounting / tracker telemetry). Defaults to
/// <see cref="NullDriverCapabilityInvoker"/> (a genuine pass-through) when none is supplied — so
/// unit tests + the F7 skeleton path behave exactly as a raw driver call. The fused Host builds a
/// real per-instance invoker via <see cref="IDriverCapabilityInvokerFactory"/> and injects it at
@@ -589,14 +589,16 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
try
{
// Route through the resilience invoker (Write pipeline: timeout, no retry by default,
// per-host breaker). An OPC UA attribute set is idempotent (writing value X twice == once),
// so a configured Write retry is safe to apply; the Write tier default is RetryCount=0, so
// this stays behavior-neutral until an operator opts in via ResilienceConfig. The outer cts
// is retained as a hard backstop above the (typically tighter) tier timeout.
// Route through the resilience invoker (Write pipeline: timeout, per-host breaker). Writes
// default to NON-idempotent (03/S12): a timed-out write may already have committed at the
// device, and replaying a command-shaped write (pulse coil, counter increment, Galaxy
// supervisory write) can duplicate an irreversible field action. The invoker's no-retry arm is
// therefore authoritative regardless of any configured Write retry. Per-tag opt-in to retries via
// WriteIdempotentAttribute is the unshipped forward path (see WriteIdempotentAttribute). The outer
// cts is retained as a hard backstop above the (typically tighter) tier timeout.
var results = await _invoker.ExecuteWriteAsync(
ResolveHost(msg.TagId),
isIdempotent: true,
isIdempotent: false,
async ct => await writable.WriteAsync(request, ct),
cts.Token);
if (results is { Count: 1 } && IsGoodStatus(results[0].StatusCode))
@@ -12,7 +12,7 @@ namespace ZB.MOM.WW.OtOpcUa.Analyzers;
/// Diagnostic analyzer that flags direct invocations of Phase 6.1-wrapped driver-capability
/// methods when the call is NOT already running inside a <c>CapabilityInvoker.ExecuteAsync</c>
/// or <c>CapabilityInvoker.ExecuteWriteAsync</c> lambda. The wrapping is what gives us
/// per-host breaker isolation, retry semantics, bulkhead-depth accounting, and alarm-ack
/// per-host breaker isolation, retry semantics, in-flight-depth accounting, and alarm-ack
/// idempotence guards — raw calls bypass all of that.
/// </summary>
/// <remarks>
@@ -102,7 +102,7 @@ public sealed class UnwrappedCapabilityCallAnalyzer : DiagnosticAnalyzer
private static readonly DiagnosticDescriptor Rule = new(
id: DiagnosticId,
title: "Driver capability call must be wrapped in CapabilityInvoker",
messageFormat: "Call to '{0}' is not wrapped in CapabilityInvoker.ExecuteAsync / ExecuteWriteAsync. Without the wrapping, Phase 6.1 resilience (retry, breaker, bulkhead, tracker telemetry) is bypassed for this call.",
messageFormat: "Call to '{0}' is not wrapped in CapabilityInvoker.ExecuteAsync / ExecuteWriteAsync. Without the wrapping, Phase 6.1 resilience (retry, breaker, tracker telemetry) is bypassed for this call.",
category: "OtOpcUa.Resilience",
defaultSeverity: DiagnosticSeverity.Warning,
isEnabledByDefault: true,