Merge R2-02 ResilienceConfig hardening (arch-review round 2) [PR #431]
Findings 01/S-6 (High — clamp hostile ResilienceConfig, no ValidationException driver-brick), 01/U-6 (delete dead bulkhead knob), 01/S-8=03/S12 (Write/Ack non-idempotent + retry->0), 04/C-7 (JsonObject round-trip preserves unknown keys), 01/S-7=03/S13 (per-Create options-generation kills respawn cache race). T15/T18 deferred. STATUS.md conflict resolved additively. Build clean.
This commit is contained in:
@@ -35,8 +35,6 @@ public sealed class DriverInstance
|
||||
/// Null = use the driver's tier defaults. When populated, expected shape:
|
||||
/// <code>
|
||||
/// {
|
||||
/// "bulkheadMaxConcurrent": 16,
|
||||
/// "bulkheadMaxQueue": 64,
|
||||
/// "capabilityPolicies": {
|
||||
/// "Read": { "timeoutSeconds": 5, "retryCount": 5, "breakerFailureThreshold": 3 },
|
||||
/// "Write": { "timeoutSeconds": 5, "retryCount": 0, "breakerFailureThreshold": 5 }
|
||||
|
||||
@@ -26,7 +26,9 @@ public sealed class DriverInstanceResilienceStatus
|
||||
/// <summary>Rolling count of consecutive Polly pipeline failures for this (instance, host).</summary>
|
||||
public int ConsecutiveFailures { get; set; }
|
||||
|
||||
/// <summary>Current Polly bulkhead depth (in-flight calls) for this (instance, host).</summary>
|
||||
/// <summary>In-flight capability-call depth for this (instance, host). Formerly fed a Polly
|
||||
/// concurrency-limiter strategy that was removed (R2-02/01/U-6); the column name is retained to avoid a
|
||||
/// migration on a reader-less entity — it now carries the tracker's in-flight-call count.</summary>
|
||||
public int CurrentBulkheadDepth { get; set; }
|
||||
|
||||
/// <summary>Most recent process recycle time (Tier C only; null for in-process tiers).</summary>
|
||||
|
||||
@@ -2,7 +2,7 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Abstraction over the Phase 6.1 resilience pipeline that wraps a single driver instance's
|
||||
/// capability calls (retry / circuit-breaker / bulkhead / tracker telemetry). The Runtime
|
||||
/// capability calls (retry / circuit-breaker / in-flight accounting / tracker telemetry). The Runtime
|
||||
/// dispatch layer (<c>DriverInstanceActor</c>) consumes this interface instead of the concrete
|
||||
/// <c>CapabilityInvoker</c> so the actor assembly does NOT pull in
|
||||
/// <c>ZB.MOM.WW.OtOpcUa.Core</c> (which drags in Polly + driver hosting) — the same boundary
|
||||
@@ -15,7 +15,7 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
/// this interface without adapters. Callers pass the resolved <c>hostName</c> per call (a
|
||||
/// multi-host driver resolves it from the tag reference via <see cref="IPerCallHostResolver"/>;
|
||||
/// a single-host driver passes its <see cref="IDriver.DriverInstanceId"/>) so per-host breaker
|
||||
/// isolation + bulkhead-depth accounting key on the right target.
|
||||
/// isolation + in-flight-depth accounting key on the right target.
|
||||
/// </remarks>
|
||||
public interface IDriverCapabilityInvoker
|
||||
{
|
||||
@@ -48,7 +48,10 @@ public interface IDriverCapabilityInvoker
|
||||
/// Execute a <see cref="DriverCapability.Write"/> call honoring idempotence semantics — when
|
||||
/// <paramref name="isIdempotent"/> is <c>false</c>, retries are disabled regardless of the
|
||||
/// configured policy; when <c>true</c>, the call runs through the Write pipeline which may
|
||||
/// retry if the tier configuration permits.
|
||||
/// retry if the tier configuration permits. <b>The current production default is <c>false</c></b>
|
||||
/// (R2-02/S-8): <c>DriverInstanceActor.HandleWriteAsync</c> passes <c>isIdempotent: false</c> so a
|
||||
/// command-shaped write is never replayed; per-tag opt-in via <see cref="WriteIdempotentAttribute"/>
|
||||
/// is the unshipped forward path.
|
||||
/// </summary>
|
||||
/// <typeparam name="TResult">Return type of the underlying driver write call.</typeparam>
|
||||
/// <param name="hostName">The resolved host name for pipeline keying + status tracking.</param>
|
||||
|
||||
@@ -28,7 +28,7 @@ public interface IPerCallHostResolver
|
||||
/// <summary>
|
||||
/// Resolve the host name for the given driver-side full reference. Returned value is
|
||||
/// used as the <c>hostName</c> argument to the Phase 6.1 <c>CapabilityInvoker</c> so
|
||||
/// per-host breaker isolation + per-host bulkhead accounting both kick in.
|
||||
/// per-host breaker isolation + per-host in-flight accounting both kick in.
|
||||
/// </summary>
|
||||
/// <param name="fullReference">The full reference of the tag or resource.</param>
|
||||
/// <returns>The host name responsible for serving the reference.</returns>
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Legal ranges for per-capability resilience policy values. Single source of truth shared by
|
||||
/// <c>DriverResilienceOptionsParser</c> (clamp + diagnostic), <c>DriverResiliencePipelineBuilder</c>
|
||||
/// (last-resort clamp so a pipeline build can never throw Polly's ValidationException), and the
|
||||
/// AdminUI <c>ResilienceFormModel</c> (authoring-time validation messages). Derived from
|
||||
/// Polly.Core 8.6.6's option validation: <c>TimeoutStrategyOptions.Timeout</c> ∈ [10 ms, 24 h];
|
||||
/// <c>CircuitBreakerStrategyOptions.MinimumThroughput</c> ≥ 2.
|
||||
/// </summary>
|
||||
public static class ResiliencePolicyRanges
|
||||
{
|
||||
/// <summary>Minimum per-call timeout. Whole-second granularity puts the floor at 1 s (Polly's is 10 ms).</summary>
|
||||
public const int MinTimeoutSeconds = 1;
|
||||
|
||||
/// <summary>Maximum per-call timeout — Polly's TimeoutStrategyOptions ceiling (24 h).</summary>
|
||||
public const int MaxTimeoutSeconds = 86_400;
|
||||
|
||||
/// <summary>Retry floor; 0 = no retry strategy is added to the pipeline.</summary>
|
||||
public const int MinRetryCount = 0;
|
||||
|
||||
/// <summary>Operational retry cap (Polly itself allows int.MaxValue; 100 exponential-backoff attempts ≈ minutes).</summary>
|
||||
public const int MaxRetryCount = 100;
|
||||
|
||||
/// <summary>Breaker floor; 0 = no breaker strategy is added (the Tier C posture).</summary>
|
||||
public const int MinBreakerFailureThreshold = 0;
|
||||
|
||||
/// <summary>Polly's MinimumThroughput floor — an enabled breaker needs a threshold of at least 2.</summary>
|
||||
public const int MinEnabledBreakerFailureThreshold = 2;
|
||||
|
||||
/// <summary>Sanity cap on the breaker threshold (Polly has no ceiling; beyond this it can never trip in the 30 s window).</summary>
|
||||
public const int MaxBreakerFailureThreshold = 10_000;
|
||||
}
|
||||
@@ -9,9 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
/// <remarks>
|
||||
/// Applied to tag-definition POCOs
|
||||
/// (e.g. <c>ModbusTagDefinition</c>, <c>S7TagDefinition</c>, OPC UA client tag rows) at the
|
||||
/// property or record level. The <c>CapabilityInvoker</c> in <c>ZB.MOM.WW.OtOpcUa.Core.Resilience</c>
|
||||
/// reads this attribute via reflection once at driver-init time and caches the result; no
|
||||
/// per-write reflection cost.
|
||||
/// property or record level. <b>Reserved for the per-tag opt-in; NOT yet consulted</b> (R2-02/S-8) —
|
||||
/// the dispatch site (<c>DriverInstanceActor.HandleWriteAsync</c>) currently treats every write as
|
||||
/// non-idempotent, so no tag retries a write today. Threading tag-definition metadata to the dispatch
|
||||
/// seam is the documented forward path; when it ships, the <c>CapabilityInvoker</c> non-idempotent arm
|
||||
/// and the parser's Write/AlarmAcknowledge no-retry clamp are the two sites that relax.
|
||||
/// </remarks>
|
||||
[AttributeUsage(AttributeTargets.Property | AttributeTargets.Class | AttributeTargets.Struct, AllowMultiple = false, Inherited = true)]
|
||||
public sealed class WriteIdempotentAttribute : Attribute
|
||||
|
||||
@@ -52,7 +52,7 @@ public sealed class AlarmSurfaceInvoker
|
||||
|
||||
/// <summary>
|
||||
/// Subscribe to alarm events for a set of source node ids, fanning out by resolved host
|
||||
/// so per-host breakers / bulkheads apply. Returns one handle per host — callers that
|
||||
/// so per-host breakers apply. Returns one handle per host — callers that
|
||||
/// don't care about per-host separation may concatenate them. Each returned handle wraps
|
||||
/// the driver's opaque handle together with its resolved host so <see cref="UnsubscribeAsync"/>
|
||||
/// routes through the same host's pipeline that the subscription was created on.
|
||||
|
||||
@@ -28,6 +28,8 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
|
||||
private readonly string _driverType;
|
||||
private readonly Func<DriverResilienceOptions> _optionsAccessor;
|
||||
private readonly DriverResilienceStatusTracker? _statusTracker;
|
||||
private readonly long _optionsGeneration;
|
||||
private DriverResilienceOptions? _noRetryWriteOptions;
|
||||
|
||||
/// <summary>
|
||||
/// Construct an invoker for one driver instance.
|
||||
@@ -39,13 +41,19 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
|
||||
/// pipeline-invalidate can take effect without restarting the invoker.
|
||||
/// </param>
|
||||
/// <param name="driverType">Driver type name for structured-log enrichment (e.g. <c>"Modbus"</c>).</param>
|
||||
/// <param name="statusTracker">Optional resilience-status tracker. When wired, every capability call records start/complete so Admin <c>/hosts</c> can surface <see cref="ResilienceStatusSnapshot.CurrentInFlight"/> as the bulkhead-depth proxy.</param>
|
||||
/// <param name="statusTracker">Optional resilience-status tracker. When wired, every capability call records start/complete so Admin <c>/hosts</c> can surface <see cref="ResilienceStatusSnapshot.CurrentInFlight"/> as the in-flight-call-depth proxy.</param>
|
||||
/// <param name="optionsGeneration">
|
||||
/// Options epoch for this invoker (01/S-7). Threaded into every pipeline-cache lookup so a lingering
|
||||
/// old child's late re-cache can never poison a newer invoker's pipeline. Defaults to 0 so existing
|
||||
/// callers/tests are unaffected.
|
||||
/// </param>
|
||||
public CapabilityInvoker(
|
||||
DriverResiliencePipelineBuilder builder,
|
||||
string driverInstanceId,
|
||||
Func<DriverResilienceOptions> optionsAccessor,
|
||||
string driverType = "Unknown",
|
||||
DriverResilienceStatusTracker? statusTracker = null)
|
||||
DriverResilienceStatusTracker? statusTracker = null,
|
||||
long optionsGeneration = 0)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(builder);
|
||||
ArgumentNullException.ThrowIfNull(optionsAccessor);
|
||||
@@ -55,6 +63,7 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
|
||||
_driverType = driverType;
|
||||
_optionsAccessor = optionsAccessor;
|
||||
_statusTracker = statusTracker;
|
||||
_optionsGeneration = optionsGeneration;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -116,27 +125,44 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker
|
||||
|
||||
if (!isIdempotent)
|
||||
{
|
||||
// Snapshot the options exactly once per call — invoking _optionsAccessor twice can
|
||||
// (a) observe two different snapshots if an Admin edit lands between them and
|
||||
// (b) wastes an allocation on the per-write hot path (Phase 6.1 1% pipeline budget).
|
||||
var snapshot = _optionsAccessor();
|
||||
var noRetryOptions = snapshot with
|
||||
// Build the no-retry options snapshot ONCE per invoker lifetime (01/P-4). The options are
|
||||
// immutable for the invoker's lifetime — a ResilienceConfig change respawns the driver child,
|
||||
// which constructs a fresh invoker (#13 respawn-on-change) — so lazily caching the snapshot is
|
||||
// safe and removes the per-write allocation the previous once-per-call snapshot incurred. The
|
||||
// null-check-assign tolerates a benign first-call race (both threads build an equivalent record).
|
||||
var noRetryOptions = _noRetryWriteOptions ??= BuildNoRetryWriteOptions();
|
||||
var pipeline = _builder.GetOrCreate(_driverInstanceId, $"{hostName}::non-idempotent", DriverCapability.Write, noRetryOptions, _optionsGeneration);
|
||||
// Record in-flight accounting on the CALLER's host (the "::non-idempotent" suffix is a
|
||||
// pipeline-cache key, not an operator-facing host) so Admin /hosts sees write depth too.
|
||||
_statusTracker?.RecordCallStart(_driverInstanceId, hostName);
|
||||
try
|
||||
{
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
using (LogContextEnricher.Push(_driverInstanceId, _driverType, DriverCapability.Write, LogContextEnricher.NewCorrelationId()))
|
||||
{
|
||||
[DriverCapability.Write] = snapshot.Resolve(DriverCapability.Write) with { RetryCount = 0 },
|
||||
},
|
||||
};
|
||||
var pipeline = _builder.GetOrCreate(_driverInstanceId, $"{hostName}::non-idempotent", DriverCapability.Write, noRetryOptions);
|
||||
using (LogContextEnricher.Push(_driverInstanceId, _driverType, DriverCapability.Write, LogContextEnricher.NewCorrelationId()))
|
||||
return await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
return await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false);
|
||||
_statusTracker?.RecordCallComplete(_driverInstanceId, hostName);
|
||||
}
|
||||
}
|
||||
|
||||
return await ExecuteAsync(DriverCapability.Write, hostName, callSite, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private DriverResilienceOptions BuildNoRetryWriteOptions()
|
||||
{
|
||||
var snapshot = _optionsAccessor();
|
||||
return snapshot with
|
||||
{
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Write] = snapshot.Resolve(DriverCapability.Write) with { RetryCount = 0 },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private ResiliencePipeline ResolvePipeline(DriverCapability capability, string hostName) =>
|
||||
_builder.GetOrCreate(_driverInstanceId, hostName, capability, _optionsAccessor());
|
||||
_builder.GetOrCreate(_driverInstanceId, hostName, capability, _optionsAccessor(), _optionsGeneration);
|
||||
}
|
||||
|
||||
@@ -19,11 +19,13 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
///
|
||||
/// <para>Options are snapshotted once per <see cref="Create"/> (a driver actor's options are fixed
|
||||
/// for its lifetime — a ResilienceConfig change respawns the child), so the invoker's per-call
|
||||
/// options accessor is allocation-free on the hot path. Because the pipeline builder caches
|
||||
/// pipelines keyed on <c>(instance, host, capability)</c> and <b>ignores options on a cache hit</b>,
|
||||
/// <see cref="Create"/> first <see cref="DriverResiliencePipelineBuilder.Invalidate"/>s any pipelines
|
||||
/// cached for the instance so a respawn with changed options rebuilds them (rather than silently
|
||||
/// reusing the stale pipeline). On a first spawn this removes nothing.</para>
|
||||
/// options accessor is allocation-free on the hot path. The pipeline builder caches pipelines keyed on
|
||||
/// <c>(instance, host, capability, generation)</c> and ignores options on a cache hit; each
|
||||
/// <see cref="Create"/> stamps a fresh monotonic <b>generation</b> (01/S-7) so the new invoker can only
|
||||
/// read pipelines it built — a lingering old child's post-invalidate re-cache lands under its own older
|
||||
/// generation and is unreachable. <see cref="Create"/> still
|
||||
/// <see cref="DriverResiliencePipelineBuilder.Invalidate"/>s the instance's pipelines as cleanup (bounds
|
||||
/// the cache), but correctness now rests on the generation key, not the invalidate timing.</para>
|
||||
/// </remarks>
|
||||
public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFactory
|
||||
{
|
||||
@@ -31,6 +33,7 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
||||
private readonly DriverResilienceStatusTracker _statusTracker;
|
||||
private readonly Func<string, DriverTier> _tierResolver;
|
||||
private readonly ILogger? _logger;
|
||||
private long _generation;
|
||||
|
||||
/// <summary>Construct the factory over the shared resilience infrastructure.</summary>
|
||||
/// <param name="builder">Process-singleton Polly pipeline builder (shared pipeline cache).</param>
|
||||
@@ -65,9 +68,14 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
||||
driverInstanceId, driverType, diagnostic);
|
||||
}
|
||||
|
||||
// Drop any pipelines cached for this instance under the PREVIOUS options — the builder keys on
|
||||
// (instance, host, capability) and reuses a cached pipeline regardless of options, so a respawn
|
||||
// with a changed ResilienceConfig would otherwise keep serving the stale pipeline. No-op on first spawn.
|
||||
// Stamp a fresh options generation for this invoker. The generation is part of the pipeline-cache
|
||||
// key (01/S-7), so this invoker can only ever read pipelines it built — a lingering old child's
|
||||
// late re-cache lands under its OWN (older) generation and is unreachable here. Correctness rests on
|
||||
// the epoch key, not on the timing of the invalidate below.
|
||||
var generation = Interlocked.Increment(ref _generation);
|
||||
|
||||
// Cleanup (no longer the correctness mechanism): drop this instance's cached pipelines across all
|
||||
// generations so lingering old-generation entries don't accumulate. No-op on first spawn.
|
||||
_builder.Invalidate(driverInstanceId);
|
||||
|
||||
return new CapabilityInvoker(
|
||||
@@ -75,6 +83,7 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
||||
driverInstanceId,
|
||||
optionsAccessor: () => options,
|
||||
driverType: driverType,
|
||||
statusTracker: _statusTracker);
|
||||
statusTracker: _statusTracker,
|
||||
optionsGeneration: generation);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,15 +19,6 @@ public sealed record DriverResilienceOptions
|
||||
public IReadOnlyDictionary<DriverCapability, CapabilityPolicy> CapabilityPolicies { get; init; }
|
||||
= new Dictionary<DriverCapability, CapabilityPolicy>();
|
||||
|
||||
/// <summary>Bulkhead (max concurrent in-flight calls) for every capability. Default 32.</summary>
|
||||
public int BulkheadMaxConcurrent { get; init; } = 32;
|
||||
|
||||
/// <summary>
|
||||
/// Bulkhead queue depth. Zero = no queueing; overflow fails fast with
|
||||
/// <c>BulkheadRejectedException</c>. Default 64.
|
||||
/// </summary>
|
||||
public int BulkheadMaxQueue { get; init; } = 64;
|
||||
|
||||
/// <summary>
|
||||
/// Periodic scheduled recycle interval for Tier C out-of-process hosts, in seconds.
|
||||
/// Null (the default) = no scheduled recycle; the driver's Host process keeps running
|
||||
@@ -69,8 +60,10 @@ public sealed record DriverResilienceOptions
|
||||
|
||||
/// <summary>
|
||||
/// Per-tier per-capability default policy table, per the Phase 6.1
|
||||
/// Stream A.2 specification. Retries skipped on <see cref="DriverCapability.Write"/> and
|
||||
/// <see cref="DriverCapability.AlarmAcknowledge"/> regardless of tier.
|
||||
/// Stream A.2 specification. The parser enforces no-retry on
|
||||
/// <see cref="DriverCapability.Write"/> and <see cref="DriverCapability.AlarmAcknowledge"/>
|
||||
/// as an invariant (R2-02/S-8) — a JSON retryCount override on those capabilities is forced back
|
||||
/// to 0; per-tag opt-in via <see cref="WriteIdempotentAttribute"/> is the future relaxation.
|
||||
/// </summary>
|
||||
/// <param name="tier">The driver tier to get defaults for.</param>
|
||||
/// <returns>The default policy dictionary for the specified tier.</returns>
|
||||
|
||||
@@ -13,8 +13,6 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
/// <para>Example JSON shape per Phase 6.1 Stream A.2:</para>
|
||||
/// <code>
|
||||
/// {
|
||||
/// "bulkheadMaxConcurrent": 16,
|
||||
/// "bulkheadMaxQueue": 64,
|
||||
/// "capabilityPolicies": {
|
||||
/// "Read": { "timeoutSeconds": 5, "retryCount": 5, "breakerFailureThreshold": 3 },
|
||||
/// "Write": { "timeoutSeconds": 5, "retryCount": 0, "breakerFailureThreshold": 5 }
|
||||
@@ -83,15 +81,16 @@ public static class DriverResilienceOptionsParser
|
||||
{
|
||||
if (!Enum.TryParse<DriverCapability>(capName, ignoreCase: true, out var capability))
|
||||
{
|
||||
parseDiagnostic ??= $"Unknown capability '{capName}' in ResilienceConfig; skipped.";
|
||||
AppendDiagnostic(ref parseDiagnostic, $"Unknown capability '{capName}' in ResilienceConfig; skipped.");
|
||||
continue;
|
||||
}
|
||||
|
||||
var basePolicy = merged[capability];
|
||||
merged[capability] = new CapabilityPolicy(
|
||||
var mergedPolicy = new CapabilityPolicy(
|
||||
TimeoutSeconds: overridePolicy.TimeoutSeconds ?? basePolicy.TimeoutSeconds,
|
||||
RetryCount: overridePolicy.RetryCount ?? basePolicy.RetryCount,
|
||||
BreakerFailureThreshold: overridePolicy.BreakerFailureThreshold ?? basePolicy.BreakerFailureThreshold);
|
||||
merged[capability] = ClampPolicy(capability, mergedPolicy, basePolicy, ref parseDiagnostic);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,9 +101,9 @@ public static class DriverResilienceOptionsParser
|
||||
if (shape.RecycleIntervalSeconds is int secs)
|
||||
{
|
||||
if (secs <= 0)
|
||||
parseDiagnostic ??= $"RecycleIntervalSeconds must be positive; got {secs} — ignoring.";
|
||||
AppendDiagnostic(ref parseDiagnostic, $"RecycleIntervalSeconds must be positive; got {secs} — ignoring.");
|
||||
else if (tier != DriverTier.C)
|
||||
parseDiagnostic ??= $"RecycleIntervalSeconds is Tier C only; Tier {tier} driver will not scheduled-recycle.";
|
||||
AppendDiagnostic(ref parseDiagnostic, $"RecycleIntervalSeconds is Tier C only; Tier {tier} driver will not scheduled-recycle.");
|
||||
else
|
||||
recycleIntervalSeconds = secs;
|
||||
}
|
||||
@@ -113,18 +112,90 @@ public static class DriverResilienceOptionsParser
|
||||
{
|
||||
Tier = tier,
|
||||
CapabilityPolicies = merged,
|
||||
BulkheadMaxConcurrent = shape.BulkheadMaxConcurrent ?? baseOptions.BulkheadMaxConcurrent,
|
||||
BulkheadMaxQueue = shape.BulkheadMaxQueue ?? baseOptions.BulkheadMaxQueue,
|
||||
RecycleIntervalSeconds = recycleIntervalSeconds,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clamp a merged per-capability override to the legal <see cref="ResiliencePolicyRanges"/> so a
|
||||
/// downstream Polly pipeline build can never throw <c>ValidationException</c> (01/S-6). Each clamp
|
||||
/// appends a diagnostic naming the field + what happened. A non-positive timeout snaps to the
|
||||
/// capability's <paramref name="tierDefault"/> (a nonsense value, not merely an extreme — snapping
|
||||
/// to 1 s could be a 30× surprise on a Discover policy); everything else clamps to the nearest legal
|
||||
/// value. Preserves maximum operator intent while never bricking.
|
||||
/// </summary>
|
||||
private static CapabilityPolicy ClampPolicy(
|
||||
DriverCapability capability,
|
||||
CapabilityPolicy merged,
|
||||
CapabilityPolicy tierDefault,
|
||||
ref string? diagnostic)
|
||||
{
|
||||
var timeout = merged.TimeoutSeconds;
|
||||
if (timeout <= 0)
|
||||
{
|
||||
AppendDiagnostic(ref diagnostic,
|
||||
$"{capability}.timeoutSeconds {timeout} is non-positive; using the tier default ({tierDefault.TimeoutSeconds}s).");
|
||||
timeout = tierDefault.TimeoutSeconds;
|
||||
}
|
||||
else if (timeout > ResiliencePolicyRanges.MaxTimeoutSeconds)
|
||||
{
|
||||
AppendDiagnostic(ref diagnostic,
|
||||
$"{capability}.timeoutSeconds {timeout} exceeds the max ({ResiliencePolicyRanges.MaxTimeoutSeconds}s); clamped.");
|
||||
timeout = ResiliencePolicyRanges.MaxTimeoutSeconds;
|
||||
}
|
||||
|
||||
var retry = merged.RetryCount;
|
||||
if (retry < ResiliencePolicyRanges.MinRetryCount)
|
||||
{
|
||||
AppendDiagnostic(ref diagnostic, $"{capability}.retryCount {retry} is negative; clamped to {ResiliencePolicyRanges.MinRetryCount}.");
|
||||
retry = ResiliencePolicyRanges.MinRetryCount;
|
||||
}
|
||||
else if (retry > ResiliencePolicyRanges.MaxRetryCount)
|
||||
{
|
||||
AppendDiagnostic(ref diagnostic, $"{capability}.retryCount {retry} exceeds the cap ({ResiliencePolicyRanges.MaxRetryCount}); clamped.");
|
||||
retry = ResiliencePolicyRanges.MaxRetryCount;
|
||||
}
|
||||
|
||||
// Spec invariant (01/S-8 = 03/S12): Write and AlarmAcknowledge are write-shaped — a duplicate is
|
||||
// not harmless (pulse coils, counter increments, Galaxy supervisory writes, alarm acks). Retries are
|
||||
// "skipped regardless of tier" per Phase 6.1; the tier defaults are already 0, so only an override can
|
||||
// differ — force it back to 0 here so the JSON→options funnel enforces the invariant. When per-tag
|
||||
// opt-in via WriteIdempotentAttribute ships, this is the one line to relax.
|
||||
if ((capability == DriverCapability.Write || capability == DriverCapability.AlarmAcknowledge) && retry > 0)
|
||||
{
|
||||
AppendDiagnostic(ref diagnostic,
|
||||
$"{capability} never retries (writes are treated as non-idempotent until per-tag opt-in ships); retryCount override ignored.");
|
||||
retry = 0;
|
||||
}
|
||||
|
||||
var breaker = merged.BreakerFailureThreshold;
|
||||
if (breaker < ResiliencePolicyRanges.MinBreakerFailureThreshold)
|
||||
{
|
||||
AppendDiagnostic(ref diagnostic, $"{capability}.breakerFailureThreshold {breaker} is negative; breaker disabled.");
|
||||
breaker = ResiliencePolicyRanges.MinBreakerFailureThreshold;
|
||||
}
|
||||
else if (breaker == 1)
|
||||
{
|
||||
AppendDiagnostic(ref diagnostic,
|
||||
$"{capability}.breakerFailureThreshold 1 is below Polly's minimum throughput; raised to {ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold} (closest to open-on-first-failure intent).");
|
||||
breaker = ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold;
|
||||
}
|
||||
else if (breaker > ResiliencePolicyRanges.MaxBreakerFailureThreshold)
|
||||
{
|
||||
AppendDiagnostic(ref diagnostic,
|
||||
$"{capability}.breakerFailureThreshold {breaker} exceeds the cap ({ResiliencePolicyRanges.MaxBreakerFailureThreshold}); clamped.");
|
||||
breaker = ResiliencePolicyRanges.MaxBreakerFailureThreshold;
|
||||
}
|
||||
|
||||
return merged with { TimeoutSeconds = timeout, RetryCount = retry, BreakerFailureThreshold = breaker };
|
||||
}
|
||||
|
||||
/// <summary>Append a diagnostic to the running channel (joined with "; ") so multiple clamps all surface.</summary>
|
||||
private static void AppendDiagnostic(ref string? diagnostic, string message)
|
||||
=> diagnostic = diagnostic is null ? message : $"{diagnostic}; {message}";
|
||||
|
||||
private sealed class ResilienceConfigShape
|
||||
{
|
||||
/// <summary>Gets or sets the maximum concurrent bulkhead requests.</summary>
|
||||
public int? BulkheadMaxConcurrent { get; set; }
|
||||
/// <summary>Gets or sets the maximum bulkhead queue size.</summary>
|
||||
public int? BulkheadMaxQueue { get; set; }
|
||||
/// <summary>Gets or sets the scheduled recycle interval in seconds.</summary>
|
||||
public int? RecycleIntervalSeconds { get; set; }
|
||||
/// <summary>Gets or sets the per-capability resilience policies.</summary>
|
||||
|
||||
@@ -15,11 +15,17 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Per-device isolation. Composition from outside-in:
|
||||
/// <b>Timeout → Retry (when capability permits) → Circuit Breaker (when tier permits) → Bulkhead</b>.
|
||||
/// <b>Timeout → Retry (when capability permits) → Circuit Breaker (when tier permits)</b>.
|
||||
///
|
||||
/// <para>Pipeline resolution is lock-free on the hot path: the inner
|
||||
/// <see cref="ConcurrentDictionary{TKey,TValue}"/> caches a <see cref="ResiliencePipeline"/> per key;
|
||||
/// first-call cost is one <see cref="ResiliencePipelineBuilder"/>.Build. Thereafter reads are O(1).</para>
|
||||
///
|
||||
/// <para><b>Never-throws invariant (01/S-6):</b> <see cref="Build"/> clamps every policy value to
|
||||
/// <see cref="ResiliencePolicyRanges"/> before constructing strategy options, so a hostile
|
||||
/// <see cref="DriverResilienceOptions"/> (constructed directly, bypassing the parser) can never make
|
||||
/// the memoizing <c>GetOrAdd</c> factory throw Polly's <c>ValidationException</c> — which would
|
||||
/// otherwise brick every capability call for that key.</para>
|
||||
/// </remarks>
|
||||
public sealed class DriverResiliencePipelineBuilder
|
||||
{
|
||||
@@ -33,7 +39,7 @@ public sealed class DriverResiliencePipelineBuilder
|
||||
/// <param name="statusTracker">When non-null, every built pipeline wires Polly telemetry into
|
||||
/// the tracker — retries increment <c>ConsecutiveFailures</c>, breaker-open stamps
|
||||
/// <c>LastBreakerOpenUtc</c>, breaker-close resets failures. Feeds Admin <c>/hosts</c> +
|
||||
/// the Polly bulkhead-depth column. Absent tracker means no telemetry (unit tests +
|
||||
/// the in-flight-call-depth column. Absent tracker means no telemetry (unit tests +
|
||||
/// deployments that don't care about resilience observability).</param>
|
||||
/// <param name="logger">When non-null, retry / circuit-breaker-open / breaker-close events are
|
||||
/// logged (instance + host + capability + attempt/exception). This is the operator-facing
|
||||
@@ -63,23 +69,35 @@ public sealed class DriverResiliencePipelineBuilder
|
||||
/// </param>
|
||||
/// <param name="capability">Which capability surface is being called.</param>
|
||||
/// <param name="options">Per-driver-instance options (tier + per-capability overrides).</param>
|
||||
/// <param name="optionsGeneration">
|
||||
/// Monotonic options epoch stamped by <see cref="DriverCapabilityInvokerFactory"/> per invoker
|
||||
/// (01/S-7). Part of the cache key so a lingering old child's post-invalidate re-cache lands under
|
||||
/// its OWN (old) generation — a key the new invoker's generation never reads. Defaults to 0 so
|
||||
/// every existing caller/test compiles unchanged.
|
||||
/// </param>
|
||||
/// <returns>The cached or newly built resilience pipeline for the key.</returns>
|
||||
public ResiliencePipeline GetOrCreate(
|
||||
string driverInstanceId,
|
||||
string hostName,
|
||||
DriverCapability capability,
|
||||
DriverResilienceOptions options)
|
||||
DriverResilienceOptions options,
|
||||
long optionsGeneration = 0)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(hostName);
|
||||
|
||||
var key = new PipelineKey(driverInstanceId, hostName, capability);
|
||||
var key = new PipelineKey(driverInstanceId, hostName, capability, optionsGeneration);
|
||||
return _pipelines.GetOrAdd(key, static (k, state) => Build(
|
||||
k.DriverInstanceId, k.HostName, state.capability, state.options, state.timeProvider, state.tracker, state.logger),
|
||||
(capability, options, timeProvider: _timeProvider, tracker: _statusTracker, logger: _logger));
|
||||
}
|
||||
|
||||
/// <summary>Drop cached pipelines for one driver instance (e.g. on ResilienceConfig change). Test + Admin-reload use.</summary>
|
||||
/// <summary>
|
||||
/// Drop cached pipelines for one driver instance (e.g. on ResilienceConfig change), across ALL
|
||||
/// option generations. With generation-keyed entries (01/S-7) correctness no longer depends on this
|
||||
/// sweep — a stale-epoch re-cache is already unreachable by the new generation — but it bounds the
|
||||
/// cache by clearing lingering old-generation entries on the next respawn. Test + Admin-reload use.
|
||||
/// </summary>
|
||||
/// <param name="driverInstanceId">The driver instance ID whose pipelines should be invalidated.</param>
|
||||
/// <returns>The number of pipelines removed.</returns>
|
||||
public int Invalidate(string driverInstanceId)
|
||||
@@ -108,16 +126,23 @@ public sealed class DriverResiliencePipelineBuilder
|
||||
var policy = options.Resolve(capability);
|
||||
var builder = new ResiliencePipelineBuilder { TimeProvider = timeProvider };
|
||||
|
||||
// Belt-and-braces: clamp the resolved policy to ResiliencePolicyRanges so building a pipeline can
|
||||
// NEVER throw Polly's ValidationException (01/S-6). The parser already clamps every JSON→options
|
||||
// funnel, but DriverResilienceOptions is a public record constructible by paths that never traverse
|
||||
// the parser (tests today; any future caller), so this makes "a pipeline build never throws" a
|
||||
// builder invariant. Three integer ops once per pipeline build (not per call).
|
||||
var timeoutSeconds = Math.Clamp(
|
||||
policy.TimeoutSeconds, ResiliencePolicyRanges.MinTimeoutSeconds, ResiliencePolicyRanges.MaxTimeoutSeconds);
|
||||
builder.AddTimeout(new TimeoutStrategyOptions
|
||||
{
|
||||
Timeout = TimeSpan.FromSeconds(policy.TimeoutSeconds),
|
||||
Timeout = TimeSpan.FromSeconds(timeoutSeconds),
|
||||
});
|
||||
|
||||
if (policy.RetryCount > 0)
|
||||
{
|
||||
var retryOptions = new RetryStrategyOptions
|
||||
{
|
||||
MaxRetryAttempts = policy.RetryCount,
|
||||
MaxRetryAttempts = Math.Min(policy.RetryCount, ResiliencePolicyRanges.MaxRetryCount),
|
||||
BackoffType = DelayBackoffType.Exponential,
|
||||
UseJitter = true,
|
||||
Delay = TimeSpan.FromMilliseconds(100),
|
||||
@@ -143,7 +168,7 @@ public sealed class DriverResiliencePipelineBuilder
|
||||
var breakerOptions = new CircuitBreakerStrategyOptions
|
||||
{
|
||||
FailureRatio = 1.0,
|
||||
MinimumThroughput = policy.BreakerFailureThreshold,
|
||||
MinimumThroughput = Math.Max(ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold, policy.BreakerFailureThreshold),
|
||||
SamplingDuration = TimeSpan.FromSeconds(30),
|
||||
BreakDuration = TimeSpan.FromSeconds(15),
|
||||
ShouldHandle = new PredicateBuilder().Handle<Exception>(ex => ex is not OperationCanceledException),
|
||||
@@ -175,5 +200,5 @@ public sealed class DriverResiliencePipelineBuilder
|
||||
return builder.Build();
|
||||
}
|
||||
|
||||
private readonly record struct PipelineKey(string DriverInstanceId, string HostName, DriverCapability Capability);
|
||||
private readonly record struct PipelineKey(string DriverInstanceId, string HostName, DriverCapability Capability, long Generation);
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ public sealed class DriverResilienceStatusTracker
|
||||
/// <summary>
|
||||
/// Record the entry of a capability call for this (instance, host). Increments the
|
||||
/// in-flight counter used as the <see cref="ResilienceStatusSnapshot.CurrentInFlight"/>
|
||||
/// surface (a cheap stand-in for Polly bulkhead depth). Paired with
|
||||
/// surface (in-flight capability-call depth). Paired with
|
||||
/// <see cref="RecordCallComplete"/>; callers use try/finally.
|
||||
/// </summary>
|
||||
/// <param name="driverInstanceId">The driver instance identifier.</param>
|
||||
@@ -159,7 +159,7 @@ public sealed record ResilienceStatusSnapshot
|
||||
/// <summary>
|
||||
/// In-flight capability calls against this (instance, host). Bumped on call entry +
|
||||
/// decremented on completion. Feeds <c>DriverInstanceResilienceStatus.CurrentBulkheadDepth</c>
|
||||
/// for Admin <c>/hosts</c> — a cheap proxy for the Polly bulkhead depth until the full
|
||||
/// for Admin <c>/hosts</c> — in-flight capability-call depth until the full
|
||||
/// telemetry observer lands.
|
||||
/// </summary>
|
||||
public int CurrentInFlight { get; init; }
|
||||
|
||||
@@ -14,7 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip;
|
||||
/// <para>Wire layer is libplctag 1.6.x. Per-device host addresses use
|
||||
/// the <c>ab://gateway[:port]/cip-path</c> canonical form parsed via
|
||||
/// <see cref="AbCipHostAddress.TryParse"/>; those strings become the <c>hostName</c> key
|
||||
/// for Polly bulkhead + circuit-breaker isolation.</para>
|
||||
/// for Polly per-host circuit-breaker isolation.</para>
|
||||
///
|
||||
/// <para>Tier A — in-process, shares server lifetime, no
|
||||
/// sidecar. <see cref="ReinitializeAsync"/> is the Tier-B escape hatch for recovering
|
||||
|
||||
@@ -3,7 +3,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip;
|
||||
/// <summary>
|
||||
/// Parsed <c>ab://gateway[:port]/cip-path</c> host-address string used by the AbCip driver
|
||||
/// as the <c>hostName</c> key across <see cref="Core.Abstractions.IHostConnectivityProbe"/>,
|
||||
/// <see cref="Core.Abstractions.IPerCallHostResolver"/>, and the Polly bulkhead key
|
||||
/// <see cref="Core.Abstractions.IPerCallHostResolver"/>, and the Polly per-host pipeline key
|
||||
/// <c>(DriverInstanceId, hostName)</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
|
||||
@@ -4,7 +4,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
|
||||
|
||||
/// <summary>
|
||||
/// FOCAS driver configuration. One instance supports N CNC devices. Each device gets its own
|
||||
/// <c>(DriverInstanceId, HostAddress)</c> bulkhead key at the Phase 6.1 resilience layer.
|
||||
/// <c>(DriverInstanceId, HostAddress)</c> per-host pipeline key at the Phase 6.1 resilience layer.
|
||||
/// </summary>
|
||||
public sealed class FocasDriverOptions
|
||||
{
|
||||
|
||||
+49
-9
@@ -6,13 +6,23 @@
|
||||
<p class="form-text mb-3">Blank fields use the driver type's stability-tier defaults
|
||||
(see <span class="mono">docs/v2/driver-stability.md</span>). Set only what you need to override.</p>
|
||||
|
||||
@if (_m.ParseFailed)
|
||||
{
|
||||
<div class="alert alert-danger" role="alert">
|
||||
<strong>Stored resilience config could not be parsed.</strong>
|
||||
Editing is locked so an accidental keystroke can't overwrite it. Fix the stored JSON directly,
|
||||
or discard it and start from tier defaults.
|
||||
<div class="mt-2">
|
||||
<button type="button" class="btn btn-sm btn-outline-danger" @onclick="DiscardAsync">
|
||||
Discard stored JSON and start blank
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
|
||||
<div class="row g-3">
|
||||
<div class="col-md-4"><label class="form-label">Bulkhead max concurrent</label>
|
||||
<input type="number" class="form-control form-control-sm" @bind="_m.BulkheadMaxConcurrent" @bind:after="EmitAsync" placeholder="tier default" /></div>
|
||||
<div class="col-md-4"><label class="form-label">Bulkhead max queue</label>
|
||||
<input type="number" class="form-control form-control-sm" @bind="_m.BulkheadMaxQueue" @bind:after="EmitAsync" placeholder="tier default" /></div>
|
||||
<div class="col-md-4"><label class="form-label">Recycle interval (s, Tier C only)</label>
|
||||
<input type="number" class="form-control form-control-sm" @bind="_m.RecycleIntervalSeconds" @bind:after="EmitAsync" placeholder="none" /></div>
|
||||
<input type="number" class="form-control form-control-sm" @bind="_m.RecycleIntervalSeconds" @bind:after="EmitAsync" placeholder="none" disabled="@_m.ParseFailed" /></div>
|
||||
</div>
|
||||
|
||||
<div class="table-wrap mt-3">
|
||||
@@ -22,20 +32,37 @@
|
||||
@foreach (var cap in ResilienceFormModel.Capabilities)
|
||||
{
|
||||
var row = _m.Policies[cap];
|
||||
var writeShaped = cap is "Write" or "AlarmAcknowledge";
|
||||
<tr>
|
||||
<td class="mono">@cap</td>
|
||||
<td><input type="number" class="form-control form-control-sm" @bind="row.TimeoutSeconds" @bind:after="EmitAsync" placeholder="default" /></td>
|
||||
<td><input type="number" class="form-control form-control-sm" @bind="row.RetryCount" @bind:after="EmitAsync" placeholder="default" /></td>
|
||||
<td><input type="number" class="form-control form-control-sm" @bind="row.BreakerFailureThreshold" @bind:after="EmitAsync" placeholder="default" /></td>
|
||||
<td><input type="number" class="form-control form-control-sm" @bind="row.TimeoutSeconds" @bind:after="EmitAsync" placeholder="default" disabled="@_m.ParseFailed" /></td>
|
||||
<td><input type="number" class="form-control form-control-sm" @bind="row.RetryCount" @bind:after="EmitAsync" placeholder="@(writeShaped ? "never" : "default")" disabled="@(writeShaped || _m.ParseFailed)" title="@(writeShaped ? "Writes/acks never retry — per-tag opt-in not yet available" : null)" /></td>
|
||||
<td><input type="number" class="form-control form-control-sm" @bind="row.BreakerFailureThreshold" @bind:after="EmitAsync" placeholder="default" disabled="@_m.ParseFailed" /></td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
@{ var _warnings = _m.Validate(); }
|
||||
@if (_warnings.Count > 0)
|
||||
{
|
||||
<div class="alert alert-warning mt-3" role="alert">
|
||||
<strong>Out-of-range values will be clamped to a safe value on deploy:</strong>
|
||||
<ul class="mb-0">
|
||||
@foreach (var w in _warnings)
|
||||
{
|
||||
<li>@w</li>
|
||||
}
|
||||
</ul>
|
||||
</div>
|
||||
}
|
||||
|
||||
<details class="mt-3">
|
||||
<summary class="small text-muted">Raw JSON (advanced)</summary>
|
||||
<pre class="form-control form-control-sm mono mt-2" style="white-space:pre-wrap;min-height:3rem;">@(_m.ToJson() ?? "(null — all tier defaults)")</pre>
|
||||
@* Render the bound STORED value, not the re-serialized model — shows the truth in both the
|
||||
healthy and the malformed case. *@
|
||||
<pre class="form-control form-control-sm mono mt-2" style="white-space:pre-wrap;min-height:3rem;">@(ResilienceConfig ?? "(null — all tier defaults)")</pre>
|
||||
</details>
|
||||
</div>
|
||||
</section>
|
||||
@@ -59,9 +86,22 @@
|
||||
|
||||
private async Task EmitAsync()
|
||||
{
|
||||
// Never emit while the stored config is unparseable — the section is locked, and an unrelated
|
||||
// keystroke must never silently overwrite what we couldn't read (04/C-7).
|
||||
if (_m.ParseFailed) return;
|
||||
|
||||
var json = _m.ToJson();
|
||||
_lastParsed = json;
|
||||
ResilienceConfig = json;
|
||||
await ResilienceConfigChanged.InvokeAsync(json);
|
||||
}
|
||||
|
||||
private async Task DiscardAsync()
|
||||
{
|
||||
// Explicit, visible destruction — replace the unparseable blob with a blank (tier-default) config.
|
||||
_m = new ResilienceFormModel();
|
||||
_lastParsed = null;
|
||||
ResilienceConfig = null;
|
||||
await ResilienceConfigChanged.InvokeAsync(null);
|
||||
}
|
||||
}
|
||||
|
||||
+155
-69
@@ -1,23 +1,28 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.Json.Nodes;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Mutable, all-nullable form model for the driver resilience override. Binds the typed
|
||||
/// fields in DriverResilienceSection; null/blank = "use the driver's tier default", so a
|
||||
/// blank form serializes back to null (preserving DriverInstance.ResilienceConfig = null).
|
||||
/// Emits / reads the exact override JSON shape DriverResilienceOptionsParser consumes.
|
||||
/// Mutable, all-nullable form model for the driver resilience override. Binds the typed fields in
|
||||
/// DriverResilienceSection; null/blank = "use the driver's tier default", so a blank form serializes
|
||||
/// back to null (preserving DriverInstance.ResilienceConfig = null). Emits / reads the exact override
|
||||
/// JSON shape DriverResilienceOptionsParser consumes.
|
||||
/// <para>
|
||||
/// Round-trip is <b>non-lossy</b> (04/C-7): the stored JSON is kept as a <see cref="JsonObject"/> bag,
|
||||
/// and <see cref="ToJson"/> overlays only the known fields onto that bag — so unknown top-level keys,
|
||||
/// unknown capability entries, and unknown per-policy fields all survive a load→save, mirroring the
|
||||
/// driver tag editors' preserve-unknown discipline (<c>TagConfigJson</c>). A stored blob that could not
|
||||
/// be parsed sets <see cref="ParseFailed"/> and is <b>never</b> overwritten — <see cref="ToJson"/>
|
||||
/// returns the original text verbatim.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ResilienceFormModel
|
||||
{
|
||||
public static readonly string[] Capabilities =
|
||||
["Read", "Write", "Discover", "Subscribe", "Probe", "AlarmSubscribe", "AlarmAcknowledge", "HistoryRead"];
|
||||
|
||||
/// <summary>Gets or sets the bulkhead max-concurrency override; null = use the tier default.</summary>
|
||||
public int? BulkheadMaxConcurrent { get; set; }
|
||||
/// <summary>Gets or sets the bulkhead max-queue-length override; null = use the tier default.</summary>
|
||||
public int? BulkheadMaxQueue { get; set; }
|
||||
/// <summary>Gets or sets the driver recycle-interval override, in seconds; null = use the tier default.</summary>
|
||||
public int? RecycleIntervalSeconds { get; set; }
|
||||
|
||||
@@ -26,6 +31,18 @@ public sealed class ResilienceFormModel
|
||||
public Dictionary<string, CapabilityRow> Policies { get; set; } =
|
||||
Capabilities.ToDictionary(c => c, _ => new CapabilityRow(), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// True when the stored ResilienceConfig could not be parsed as JSON. The section locks editing and
|
||||
/// <see cref="ToJson"/> returns <see cref="RawStoredJson"/> unchanged so an unrelated keystroke can
|
||||
/// never silently overwrite an unreadable column.
|
||||
/// </summary>
|
||||
public bool ParseFailed { get; private set; }
|
||||
|
||||
/// <summary>The original stored text when <see cref="ParseFailed"/> — returned verbatim by <see cref="ToJson"/>.</summary>
|
||||
public string? RawStoredJson { get; private set; }
|
||||
|
||||
private JsonObject _bag = new();
|
||||
|
||||
/// <summary>Per-capability timeout/retry/breaker override row; null fields fall back to the tier default.</summary>
|
||||
public sealed class CapabilityRow
|
||||
{
|
||||
@@ -39,88 +56,157 @@ public sealed class ResilienceFormModel
|
||||
public bool IsEmpty => TimeoutSeconds is null && RetryCount is null && BreakerFailureThreshold is null;
|
||||
}
|
||||
|
||||
private static readonly JsonSerializerOptions ReadOpts = new() { PropertyNameCaseInsensitive = true };
|
||||
private static readonly JsonSerializerOptions WriteOpts = new()
|
||||
/// <summary>
|
||||
/// Range-validate the per-capability overrides against <see cref="ResiliencePolicyRanges"/> and return
|
||||
/// one human-readable message per violation (naming capability + field + legal range). This is
|
||||
/// authoring-time <b>feedback</b>, not the enforcement layer — the parser's clamp is the authoritative
|
||||
/// guard, so the form still emits. Mirrors the driver tag editors' <c>Validate()</c> idiom (01/S-6).
|
||||
/// </summary>
|
||||
/// <returns>A list of range-violation messages; empty when every override is in range or blank.</returns>
|
||||
public IReadOnlyList<string> Validate()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
var msgs = new List<string>();
|
||||
foreach (var (cap, row) in Policies)
|
||||
{
|
||||
if (row.TimeoutSeconds is { } t &&
|
||||
(t < ResiliencePolicyRanges.MinTimeoutSeconds || t > ResiliencePolicyRanges.MaxTimeoutSeconds))
|
||||
msgs.Add($"{cap} timeout must be between {ResiliencePolicyRanges.MinTimeoutSeconds} and " +
|
||||
$"{ResiliencePolicyRanges.MaxTimeoutSeconds} seconds (got {t}).");
|
||||
|
||||
/// <summary>Parses the override JSON into a form model; malformed or blank JSON yields an empty (all-default) form.</summary>
|
||||
if (row.RetryCount is { } r &&
|
||||
(r < ResiliencePolicyRanges.MinRetryCount || r > ResiliencePolicyRanges.MaxRetryCount))
|
||||
msgs.Add($"{cap} retries must be between {ResiliencePolicyRanges.MinRetryCount} and " +
|
||||
$"{ResiliencePolicyRanges.MaxRetryCount} (got {r}).");
|
||||
|
||||
if (row.BreakerFailureThreshold is { } b &&
|
||||
(b < ResiliencePolicyRanges.MinBreakerFailureThreshold || b == 1 ||
|
||||
b > ResiliencePolicyRanges.MaxBreakerFailureThreshold))
|
||||
msgs.Add($"{cap} breaker threshold must be 0 (off) or between " +
|
||||
$"{ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold} and " +
|
||||
$"{ResiliencePolicyRanges.MaxBreakerFailureThreshold} (got {b}).");
|
||||
}
|
||||
return msgs;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse the override JSON into a form model, preserving every key in an internal bag. Malformed JSON
|
||||
/// sets <see cref="ParseFailed"/> + <see cref="RawStoredJson"/> and yields an otherwise-empty model.
|
||||
/// </summary>
|
||||
/// <param name="json">The raw resilience-override JSON, or null/blank if there is no override.</param>
|
||||
/// <returns>A populated form model, or an all-default one when <paramref name="json"/> is blank or malformed.</returns>
|
||||
/// <returns>A populated form model.</returns>
|
||||
public static ResilienceFormModel FromJson(string? json)
|
||||
{
|
||||
var model = new ResilienceFormModel();
|
||||
if (string.IsNullOrWhiteSpace(json)) return model;
|
||||
|
||||
Shape? shape;
|
||||
try { shape = JsonSerializer.Deserialize<Shape>(json, ReadOpts); }
|
||||
catch (JsonException) { return model; } // malformed -> empty form; raw view (next task) shows the text
|
||||
if (shape is null) return model;
|
||||
JsonObject bag;
|
||||
try
|
||||
{
|
||||
bag = JsonNode.Parse(json) as JsonObject ?? new JsonObject();
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
model.ParseFailed = true;
|
||||
model.RawStoredJson = json;
|
||||
return model;
|
||||
}
|
||||
|
||||
model.BulkheadMaxConcurrent = shape.BulkheadMaxConcurrent;
|
||||
model.BulkheadMaxQueue = shape.BulkheadMaxQueue;
|
||||
model.RecycleIntervalSeconds = shape.RecycleIntervalSeconds;
|
||||
if (shape.CapabilityPolicies is not null)
|
||||
foreach (var (cap, p) in shape.CapabilityPolicies)
|
||||
if (model.Policies.TryGetValue(cap, out var row))
|
||||
model._bag = bag;
|
||||
model.RecycleIntervalSeconds = GetIntOrNull(bag, "recycleIntervalSeconds");
|
||||
|
||||
if (bag.TryGetPropertyValue("capabilityPolicies", out var cpNode) && cpNode is JsonObject cp)
|
||||
{
|
||||
foreach (var (capName, polNode) in cp)
|
||||
{
|
||||
// Known capability names populate rows (reading only the known per-policy fields — unknown
|
||||
// per-policy fields stay untouched in the bag). Unknown capability entries stay in the bag.
|
||||
if (polNode is JsonObject pol && model.Policies.TryGetValue(capName, out var row))
|
||||
{
|
||||
row.TimeoutSeconds = p.TimeoutSeconds;
|
||||
row.RetryCount = p.RetryCount;
|
||||
row.BreakerFailureThreshold = p.BreakerFailureThreshold;
|
||||
row.TimeoutSeconds = GetIntOrNull(pol, "timeoutSeconds");
|
||||
row.RetryCount = GetIntOrNull(pol, "retryCount");
|
||||
row.BreakerFailureThreshold = GetIntOrNull(pol, "breakerFailureThreshold");
|
||||
}
|
||||
}
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
/// <summary>Emit only the non-null overrides; returns null when nothing is overridden.</summary>
|
||||
/// <returns>The serialized override JSON, or null when no field in this form is overridden.</returns>
|
||||
/// <summary>
|
||||
/// Overlay the model's non-null known values onto the preserved bag and serialize. Returns null when
|
||||
/// the result is empty (blank form = tier defaults). When <see cref="ParseFailed"/>, returns the
|
||||
/// original unparseable text unchanged.
|
||||
/// </summary>
|
||||
/// <returns>The serialized override JSON, the original text when parse failed, or null when nothing is set.</returns>
|
||||
public string? ToJson()
|
||||
{
|
||||
var caps = Policies
|
||||
.Where(kv => !kv.Value.IsEmpty)
|
||||
.ToDictionary(kv => kv.Key, kv => new PolicyShape
|
||||
{
|
||||
TimeoutSeconds = kv.Value.TimeoutSeconds,
|
||||
RetryCount = kv.Value.RetryCount,
|
||||
BreakerFailureThreshold = kv.Value.BreakerFailureThreshold,
|
||||
});
|
||||
if (ParseFailed) return RawStoredJson;
|
||||
|
||||
var hasAny = BulkheadMaxConcurrent is not null || BulkheadMaxQueue is not null
|
||||
|| RecycleIntervalSeconds is not null || caps.Count > 0;
|
||||
if (!hasAny) return null;
|
||||
SetOrRemoveInt(_bag, "recycleIntervalSeconds", RecycleIntervalSeconds);
|
||||
|
||||
var shape = new Shape
|
||||
var cp = _bag.TryGetPropertyValue("capabilityPolicies", out var cpNode) && cpNode is JsonObject existing
|
||||
? existing
|
||||
: null;
|
||||
|
||||
foreach (var (cap, row) in Policies)
|
||||
{
|
||||
BulkheadMaxConcurrent = BulkheadMaxConcurrent,
|
||||
BulkheadMaxQueue = BulkheadMaxQueue,
|
||||
RecycleIntervalSeconds = RecycleIntervalSeconds,
|
||||
CapabilityPolicies = caps.Count > 0 ? caps : null,
|
||||
};
|
||||
return JsonSerializer.Serialize(shape, WriteOpts);
|
||||
// Find this known capability's existing policy object (case-insensitive), else its canonical key.
|
||||
var polKey = cap;
|
||||
JsonObject? pol = null;
|
||||
if (cp is not null)
|
||||
{
|
||||
foreach (var (k, v) in cp)
|
||||
{
|
||||
if (string.Equals(k, cap, StringComparison.OrdinalIgnoreCase) && v is JsonObject vo)
|
||||
{
|
||||
polKey = k;
|
||||
pol = vo;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (row.IsEmpty)
|
||||
{
|
||||
// Clear the known fields; drop the policy entry only if no unknown fields remain.
|
||||
if (pol is not null)
|
||||
{
|
||||
pol.Remove("timeoutSeconds");
|
||||
pol.Remove("retryCount");
|
||||
pol.Remove("breakerFailureThreshold");
|
||||
if (pol.Count == 0) cp!.Remove(polKey);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (cp is null)
|
||||
{
|
||||
cp = new JsonObject();
|
||||
_bag["capabilityPolicies"] = cp;
|
||||
}
|
||||
if (pol is null)
|
||||
{
|
||||
pol = new JsonObject();
|
||||
cp[polKey] = pol;
|
||||
}
|
||||
SetOrRemoveInt(pol, "timeoutSeconds", row.TimeoutSeconds);
|
||||
SetOrRemoveInt(pol, "retryCount", row.RetryCount);
|
||||
SetOrRemoveInt(pol, "breakerFailureThreshold", row.BreakerFailureThreshold);
|
||||
}
|
||||
}
|
||||
|
||||
if (cp is not null && cp.Count == 0) _bag.Remove("capabilityPolicies");
|
||||
|
||||
return _bag.Count == 0 ? null : _bag.ToJsonString();
|
||||
}
|
||||
|
||||
/// <summary>Wire shape of the resilience-override JSON, as consumed by DriverResilienceOptionsParser.</summary>
|
||||
private sealed class Shape
|
||||
{
|
||||
/// <summary>Gets or sets the bulkhead max-concurrency override.</summary>
|
||||
public int? BulkheadMaxConcurrent { get; set; }
|
||||
/// <summary>Gets or sets the bulkhead max-queue-length override.</summary>
|
||||
public int? BulkheadMaxQueue { get; set; }
|
||||
/// <summary>Gets or sets the driver recycle-interval override, in seconds.</summary>
|
||||
public int? RecycleIntervalSeconds { get; set; }
|
||||
/// <summary>Gets or sets the per-capability overrides, keyed by capability name.</summary>
|
||||
public Dictionary<string, PolicyShape>? CapabilityPolicies { get; set; }
|
||||
}
|
||||
private static int? GetIntOrNull(JsonObject o, string name)
|
||||
=> o.TryGetPropertyValue(name, out var n) && n is JsonValue v && v.TryGetValue<int>(out var i)
|
||||
? i
|
||||
: null;
|
||||
|
||||
/// <summary>Wire shape of a single capability's timeout/retry/breaker override.</summary>
|
||||
private sealed class PolicyShape
|
||||
private static void SetOrRemoveInt(JsonObject o, string name, int? value)
|
||||
{
|
||||
/// <summary>Gets or sets the timeout override, in seconds.</summary>
|
||||
public int? TimeoutSeconds { get; set; }
|
||||
/// <summary>Gets or sets the retry-count override.</summary>
|
||||
public int? RetryCount { get; set; }
|
||||
/// <summary>Gets or sets the circuit-breaker failure-threshold override.</summary>
|
||||
public int? BreakerFailureThreshold { get; set; }
|
||||
if (value is null) o.Remove(name);
|
||||
else o[name] = JsonValue.Create(value.Value);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1627,7 +1627,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
|
||||
else
|
||||
{
|
||||
// Attach the per-instance Phase 6.1 resilience invoker so this driver's capability calls
|
||||
// route through the retry/breaker/bulkhead/telemetry pipeline. The tier defaults are layered
|
||||
// route through the retry/breaker/telemetry pipeline. The tier defaults are layered
|
||||
// with the per-instance ResilienceConfig from the artifact; the pass-through factory yields a
|
||||
// no-op invoker on nodes without the real resilience factory bound.
|
||||
var invoker = _invokerFactory.Create(spec.DriverInstanceId, spec.DriverType, spec.ResilienceConfig);
|
||||
|
||||
@@ -139,7 +139,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
private readonly IDriver _driver;
|
||||
|
||||
/// <summary>Phase 6.1 resilience seam: every guarded driver-capability call this actor makes is
|
||||
/// routed through this invoker (retry / breaker / bulkhead / tracker telemetry). Defaults to
|
||||
/// routed through this invoker (retry / breaker / in-flight accounting / tracker telemetry). Defaults to
|
||||
/// <see cref="NullDriverCapabilityInvoker"/> (a genuine pass-through) when none is supplied — so
|
||||
/// unit tests + the F7 skeleton path behave exactly as a raw driver call. The fused Host builds a
|
||||
/// real per-instance invoker via <see cref="IDriverCapabilityInvokerFactory"/> and injects it at
|
||||
@@ -589,14 +589,16 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
try
|
||||
{
|
||||
// Route through the resilience invoker (Write pipeline: timeout, no retry by default,
|
||||
// per-host breaker). An OPC UA attribute set is idempotent (writing value X twice == once),
|
||||
// so a configured Write retry is safe to apply; the Write tier default is RetryCount=0, so
|
||||
// this stays behavior-neutral until an operator opts in via ResilienceConfig. The outer cts
|
||||
// is retained as a hard backstop above the (typically tighter) tier timeout.
|
||||
// Route through the resilience invoker (Write pipeline: timeout, per-host breaker). Writes
|
||||
// default to NON-idempotent (03/S12): a timed-out write may already have committed at the
|
||||
// device, and replaying a command-shaped write (pulse coil, counter increment, Galaxy
|
||||
// supervisory write) can duplicate an irreversible field action. The invoker's no-retry arm is
|
||||
// therefore authoritative regardless of any configured Write retry. Per-tag opt-in to retries via
|
||||
// WriteIdempotentAttribute is the unshipped forward path (see WriteIdempotentAttribute). The outer
|
||||
// cts is retained as a hard backstop above the (typically tighter) tier timeout.
|
||||
var results = await _invoker.ExecuteWriteAsync(
|
||||
ResolveHost(msg.TagId),
|
||||
isIdempotent: true,
|
||||
isIdempotent: false,
|
||||
async ct => await writable.WriteAsync(request, ct),
|
||||
cts.Token);
|
||||
if (results is { Count: 1 } && IsGoodStatus(results[0].StatusCode))
|
||||
|
||||
@@ -12,7 +12,7 @@ namespace ZB.MOM.WW.OtOpcUa.Analyzers;
|
||||
/// Diagnostic analyzer that flags direct invocations of Phase 6.1-wrapped driver-capability
|
||||
/// methods when the call is NOT already running inside a <c>CapabilityInvoker.ExecuteAsync</c>
|
||||
/// or <c>CapabilityInvoker.ExecuteWriteAsync</c> lambda. The wrapping is what gives us
|
||||
/// per-host breaker isolation, retry semantics, bulkhead-depth accounting, and alarm-ack
|
||||
/// per-host breaker isolation, retry semantics, in-flight-depth accounting, and alarm-ack
|
||||
/// idempotence guards — raw calls bypass all of that.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
@@ -102,7 +102,7 @@ public sealed class UnwrappedCapabilityCallAnalyzer : DiagnosticAnalyzer
|
||||
private static readonly DiagnosticDescriptor Rule = new(
|
||||
id: DiagnosticId,
|
||||
title: "Driver capability call must be wrapped in CapabilityInvoker",
|
||||
messageFormat: "Call to '{0}' is not wrapped in CapabilityInvoker.ExecuteAsync / ExecuteWriteAsync. Without the wrapping, Phase 6.1 resilience (retry, breaker, bulkhead, tracker telemetry) is bypassed for this call.",
|
||||
messageFormat: "Call to '{0}' is not wrapped in CapabilityInvoker.ExecuteAsync / ExecuteWriteAsync. Without the wrapping, Phase 6.1 resilience (retry, breaker, tracker telemetry) is bypassed for this call.",
|
||||
category: "OtOpcUa.Resilience",
|
||||
defaultSeverity: DiagnosticSeverity.Warning,
|
||||
isEnabledByDefault: true,
|
||||
|
||||
Reference in New Issue
Block a user