Merge R2-02 ResilienceConfig hardening (arch-review round 2) [PR #431]
Findings 01/S-6 (High — clamp hostile ResilienceConfig, no ValidationException driver-brick), 01/U-6 (delete dead bulkhead knob), 01/S-8=03/S12 (Write/Ack non-idempotent + retry->0), 04/C-7 (JsonObject round-trip preserves unknown keys), 01/S-7=03/S13 (per-Create options-generation kills respawn cache race). T15/T18 deferred. STATUS.md conflict resolved additively. Build clean.
This commit is contained in:
@@ -188,6 +188,53 @@ public sealed class CapabilityInvokerTests
|
||||
"ExecuteWriteAsync's non-idempotent branch must capture the options snapshot exactly once per call");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// S-8 residual: the non-idempotent write arm must record tracker in-flight accounting (start +
|
||||
/// complete) on the caller's host, exactly like the idempotent arms — the previous arm skipped it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task NonIdempotentWrite_RecordsTrackerStartAndComplete()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
var invoker = new CapabilityInvoker(
|
||||
new DriverResiliencePipelineBuilder(),
|
||||
"drv-test",
|
||||
() => new DriverResilienceOptions { Tier = DriverTier.A },
|
||||
statusTracker: tracker);
|
||||
|
||||
await invoker.ExecuteWriteAsync(
|
||||
"host-1",
|
||||
isIdempotent: false,
|
||||
_ => ValueTask.FromResult(0),
|
||||
CancellationToken.None);
|
||||
|
||||
var snap = tracker.TryGet("drv-test", "host-1");
|
||||
snap.ShouldNotBeNull("the non-idempotent write arm must record tracker accounting on the caller's host");
|
||||
snap!.CurrentInFlight.ShouldBe(0, "in-flight must return to 0 after the write completes");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 01/P-4: the non-idempotent write arm must build the no-retry options snapshot ONCE per invoker
|
||||
/// lifetime (not per call). Two writes to the same host reuse the cached snapshot — the options
|
||||
/// accessor is invoked at most once and only one pipeline is cached.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task NonIdempotentWrite_ReusesCachedNoRetryOptions()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var accessorCalls = 0;
|
||||
var invoker = new CapabilityInvoker(
|
||||
builder,
|
||||
"drv-test",
|
||||
() => { Interlocked.Increment(ref accessorCalls); return new DriverResilienceOptions { Tier = DriverTier.A }; });
|
||||
|
||||
await invoker.ExecuteWriteAsync("host-1", isIdempotent: false, _ => ValueTask.FromResult(0), CancellationToken.None);
|
||||
await invoker.ExecuteWriteAsync("host-1", isIdempotent: false, _ => ValueTask.FromResult(0), CancellationToken.None);
|
||||
|
||||
accessorCalls.ShouldBeLessThanOrEqualTo(1, "the no-retry snapshot must be cached across calls");
|
||||
builder.CachedPipelineCount.ShouldBe(1, "two writes to the same host share one cached no-retry pipeline");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Core-009 regression — companion consistency assertion: the non-idempotent branch must
|
||||
/// not observe two different option snapshots even if the accessor's returned value changes
|
||||
|
||||
+67
@@ -136,6 +136,73 @@ public sealed class DriverCapabilityInvokerFactoryTests
|
||||
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Message.Contains("resilience config", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 01/S-6 production-wiring proof: the exact <c>DriverHostActor.SpawnChild</c> path
|
||||
/// (<c>factory.Create(id, type, hostileJson)</c>) must yield a WORKING invoker whose capability
|
||||
/// call succeeds, and must log the clamp diagnostic as a warning — a hostile ResilienceConfig can
|
||||
/// never brick the spawn path.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Create_with_out_of_range_config_yields_working_invoker()
|
||||
{
|
||||
var logger = new CapturingLogger();
|
||||
var factory = MakeFactory(new DriverResiliencePipelineBuilder(), logger: logger);
|
||||
|
||||
var invoker = factory.Create("i1", "Modbus",
|
||||
resilienceConfigJson: "{\"capabilityPolicies\":{\"Subscribe\":{\"timeoutSeconds\":0,\"breakerFailureThreshold\":1}}}");
|
||||
|
||||
var result = await invoker.ExecuteAsync(
|
||||
DriverCapability.Subscribe, "host-1", _ => ValueTask.FromResult(99), CancellationToken.None);
|
||||
|
||||
result.ShouldBe(99, "a hostile ResilienceConfig must not brick the Subscribe capability call");
|
||||
logger.Entries.ShouldContain(e =>
|
||||
e.Level == LogLevel.Warning &&
|
||||
e.Message.Contains("resilience config", StringComparison.OrdinalIgnoreCase) &&
|
||||
(e.Message.Contains("timeoutSeconds") || e.Message.Contains("breakerFailureThreshold")));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 01/S-7 production-wiring proof: a respawn interleaved with a lingering old-invoker call must still
|
||||
/// apply the new options. The old invoker (a captured reference — exactly what the old child holds)
|
||||
/// re-caches its stale pipeline AFTER the new <see cref="DriverCapabilityInvokerFactory.Create"/>'s
|
||||
/// invalidate; the new invoker must nonetheless behave per the new config. Fails without the per-Create
|
||||
/// generation stamp (both invokers share the (instance, host, capability) key and the stale entry wins).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Respawn_interleaved_with_old_invoker_call_still_applies_new_options()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var factory = MakeFactory(builder);
|
||||
|
||||
// Old invoker: Read retries (tier-A default 3). Warm its cache.
|
||||
var oldInvoker = factory.Create("d1", "Modbus", "{\"capabilityPolicies\":{\"Read\":{\"retryCount\":3}}}");
|
||||
await FailingRead(oldInvoker); // caches the retrying pipeline under the old generation
|
||||
|
||||
// Respawn with new options: Read no longer retries.
|
||||
var newInvoker = factory.Create("d1", "Modbus", "{\"capabilityPolicies\":{\"Read\":{\"retryCount\":0}}}");
|
||||
|
||||
// The lingering old child's late call lands AFTER the new Create's invalidate — re-caching the stale
|
||||
// retrying pipeline under the OLD generation.
|
||||
await FailingRead(oldInvoker);
|
||||
|
||||
// The new invoker must honor the new options (no retry) — a single attempt.
|
||||
var attempts = 0;
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await newInvoker.ExecuteAsync(
|
||||
DriverCapability.Read, "host-1",
|
||||
async _ => { attempts++; await Task.Yield(); throw new InvalidOperationException("boom"); },
|
||||
CancellationToken.None));
|
||||
|
||||
attempts.ShouldBe(1, "the new invoker must apply the new options (retryCount 0), not the stale re-cache");
|
||||
}
|
||||
|
||||
private static async Task FailingRead(IDriverCapabilityInvoker invoker) =>
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await invoker.ExecuteAsync(
|
||||
DriverCapability.Read, "host-1",
|
||||
async _ => { await Task.Yield(); throw new InvalidOperationException("boom"); },
|
||||
CancellationToken.None));
|
||||
|
||||
private sealed class CapturingLogger : ILogger
|
||||
{
|
||||
public List<(LogLevel Level, string Message)> Entries { get; } = new();
|
||||
|
||||
+164
-9
@@ -98,18 +98,23 @@ public sealed class DriverResilienceOptionsParserTests
|
||||
read.BreakerFailureThreshold.ShouldBe(tierDefault.BreakerFailureThreshold);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that bulkhead overrides are honored.</summary>
|
||||
/// <summary>
|
||||
/// 01/U-6: the deleted bulkhead knob's keys in stored JSON become ignored unknown keys — a
|
||||
/// config carrying them parses clean with no diagnostic (migration-free contract).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void BulkheadOverrides_AreHonored()
|
||||
public void BulkheadKeys_InStoredJson_AreIgnored()
|
||||
{
|
||||
var json = """
|
||||
{ "bulkheadMaxConcurrent": 100, "bulkheadMaxQueue": 500 }
|
||||
""";
|
||||
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.B, json, out _);
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.B, json, out var diag);
|
||||
|
||||
options.BulkheadMaxConcurrent.ShouldBe(100);
|
||||
options.BulkheadMaxQueue.ShouldBe(500);
|
||||
diag.ShouldBeNull();
|
||||
// Known capabilities untouched — the stale bulkhead keys are simply ignored.
|
||||
options.Resolve(DriverCapability.Read).ShouldBe(
|
||||
DriverResilienceOptions.GetTierDefaults(DriverTier.B)[DriverCapability.Read]);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that unknown capability surfaces in diagnostic but does not fail.</summary>
|
||||
@@ -133,17 +138,18 @@ public sealed class DriverResilienceOptionsParserTests
|
||||
DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read]);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that property names are case insensitive.</summary>
|
||||
/// <summary>Verifies that top-level property names are case insensitive.</summary>
|
||||
[Fact]
|
||||
public void PropertyNames_AreCaseInsensitive()
|
||||
{
|
||||
var json = """
|
||||
{ "BULKHEADMAXCONCURRENT": 42 }
|
||||
{ "RECYCLEINTERVALSECONDS": 3600 }
|
||||
""";
|
||||
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, json, out _);
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.C, json, out var diag);
|
||||
|
||||
options.BulkheadMaxConcurrent.ShouldBe(42);
|
||||
diag.ShouldBeNull();
|
||||
options.RecycleIntervalSeconds.ShouldBe(3600);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that capability name is case insensitive.</summary>
|
||||
@@ -222,4 +228,153 @@ public sealed class DriverResilienceOptionsParserTests
|
||||
options.RecycleIntervalSeconds.ShouldBeNull();
|
||||
diag.ShouldContain("must be positive");
|
||||
}
|
||||
|
||||
// ---- 01/S-6: range clamps (clamp + diagnostic, never brick) ----
|
||||
|
||||
/// <summary>A non-positive timeout is nonsense; it snaps to the capability's tier default with a diagnostic.</summary>
|
||||
[Fact]
|
||||
public void TimeoutSeconds_Zero_FallsBackToTierDefault_WithDiagnostic()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"timeoutSeconds\":0}}}", out var diag);
|
||||
|
||||
var tierDefault = DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read];
|
||||
options.Resolve(DriverCapability.Read).TimeoutSeconds.ShouldBe(tierDefault.TimeoutSeconds);
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("timeoutSeconds");
|
||||
}
|
||||
|
||||
/// <summary>A timeout above Polly's ceiling clamps to the max with a diagnostic.</summary>
|
||||
[Fact]
|
||||
public void TimeoutSeconds_AbovePollyMax_Clamped()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"timeoutSeconds\":999999}}}", out var diag);
|
||||
|
||||
options.Resolve(DriverCapability.Read).TimeoutSeconds.ShouldBe(ResiliencePolicyRanges.MaxTimeoutSeconds);
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("timeoutSeconds");
|
||||
}
|
||||
|
||||
/// <summary>A negative retry count clamps to zero with a diagnostic.</summary>
|
||||
[Fact]
|
||||
public void RetryCount_Negative_ClampedToZero()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"retryCount\":-1}}}", out var diag);
|
||||
|
||||
options.Resolve(DriverCapability.Read).RetryCount.ShouldBe(0);
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("retryCount");
|
||||
}
|
||||
|
||||
/// <summary>An absurd retry count clamps to the operational cap with a diagnostic.</summary>
|
||||
[Fact]
|
||||
public void RetryCount_AboveCap_ClampedTo100()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"retryCount\":2000000000}}}", out var diag);
|
||||
|
||||
options.Resolve(DriverCapability.Read).RetryCount.ShouldBe(ResiliencePolicyRanges.MaxRetryCount);
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("retryCount");
|
||||
}
|
||||
|
||||
/// <summary>A breaker threshold of 1 (below Polly's MinimumThroughput floor) raises to 2 with a diagnostic.</summary>
|
||||
[Fact]
|
||||
public void BreakerThreshold_One_ClampedToTwo()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"breakerFailureThreshold\":1}}}", out var diag);
|
||||
|
||||
options.Resolve(DriverCapability.Read).BreakerFailureThreshold.ShouldBe(
|
||||
ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold);
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("breakerFailureThreshold");
|
||||
}
|
||||
|
||||
/// <summary>A negative breaker threshold disables the breaker (0) with a diagnostic.</summary>
|
||||
[Fact]
|
||||
public void BreakerThreshold_Negative_DisablesBreaker()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"breakerFailureThreshold\":-5}}}", out var diag);
|
||||
|
||||
options.Resolve(DriverCapability.Read).BreakerFailureThreshold.ShouldBe(0);
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("breakerFailureThreshold");
|
||||
}
|
||||
|
||||
/// <summary>Multiple clamps in one policy all surface in the (appended) diagnostic string.</summary>
|
||||
[Fact]
|
||||
public void MultipleClamps_AllSurfaceInDiagnostic()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A,
|
||||
"{\"capabilityPolicies\":{\"Read\":{\"timeoutSeconds\":0,\"retryCount\":-1,\"breakerFailureThreshold\":1}}}",
|
||||
out var diag);
|
||||
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("timeoutSeconds");
|
||||
diag.ShouldContain("retryCount");
|
||||
diag.ShouldContain("breakerFailureThreshold");
|
||||
|
||||
var read = options.Resolve(DriverCapability.Read);
|
||||
read.TimeoutSeconds.ShouldBe(DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read].TimeoutSeconds);
|
||||
read.RetryCount.ShouldBe(0);
|
||||
read.BreakerFailureThreshold.ShouldBe(ResiliencePolicyRanges.MinEnabledBreakerFailureThreshold);
|
||||
}
|
||||
|
||||
// ---- 01/S-8 = 03/S12: write-shaped capabilities never retry ----
|
||||
|
||||
/// <summary>A Write retryCount override is forced to 0 with a diagnostic (writes are non-idempotent).</summary>
|
||||
[Fact]
|
||||
public void WriteRetryOverride_IsForcedToZero_WithDiagnostic()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"Write\":{\"retryCount\":5}}}", out var diag);
|
||||
|
||||
options.Resolve(DriverCapability.Write).RetryCount.ShouldBe(0);
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("Write never retries");
|
||||
}
|
||||
|
||||
/// <summary>An AlarmAcknowledge retryCount override is forced to 0 with a diagnostic.</summary>
|
||||
[Fact]
|
||||
public void AlarmAcknowledgeRetryOverride_IsForcedToZero_WithDiagnostic()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"AlarmAcknowledge\":{\"retryCount\":3}}}", out var diag);
|
||||
|
||||
options.Resolve(DriverCapability.AlarmAcknowledge).RetryCount.ShouldBe(0);
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("AlarmAcknowledge never retries");
|
||||
}
|
||||
|
||||
/// <summary>A Read (idempotent) retryCount override is still honored — the invariant is write-shaped only.</summary>
|
||||
[Fact]
|
||||
public void ReadRetryOverride_StillHonored()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"retryCount\":5}}}", out var diag);
|
||||
|
||||
options.Resolve(DriverCapability.Read).RetryCount.ShouldBe(5);
|
||||
diag.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>In-range overrides pass through untouched with no diagnostic.</summary>
|
||||
[Fact]
|
||||
public void InRangeOverrides_PassThrough_NoDiagnostic()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A,
|
||||
"{\"capabilityPolicies\":{\"Read\":{\"timeoutSeconds\":5,\"retryCount\":3,\"breakerFailureThreshold\":4}}}",
|
||||
out var diag);
|
||||
|
||||
diag.ShouldBeNull();
|
||||
var read = options.Resolve(DriverCapability.Read);
|
||||
read.TimeoutSeconds.ShouldBe(5);
|
||||
read.RetryCount.ShouldBe(3);
|
||||
read.BreakerFailureThreshold.ShouldBe(4);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,34 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class DriverResilienceOptionsTests
|
||||
{
|
||||
/// <summary>
|
||||
/// 01/U-6 knob-inertness guard (OVERALL theme #1): the public property set of
|
||||
/// <see cref="DriverResilienceOptions"/> must be EXACTLY the pipeline-wired knobs. Every option a
|
||||
/// parser can populate must map to a strategy the builder actually composes — a parsed-but-unapplied
|
||||
/// knob (the bulkhead genre this pass deleted) is inertness the interface-forwarding and
|
||||
/// unwrapped-dispatch guards can't catch. To add a knob: wire it in <c>DriverResiliencePipelineBuilder</c>,
|
||||
/// add a behavior test that proves it engages, THEN add it to the expected set below.
|
||||
/// (<c>RecycleIntervalSeconds</c> is itself Tier-C-dormant per 01/U-2 — a documented open question,
|
||||
/// explicitly out of this pass's scope; it stays in the expected list as the recycle scheduler, not
|
||||
/// the Polly pipeline, is its consumer.)
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Options_properties_are_exactly_the_pipeline_wired_set()
|
||||
{
|
||||
var expected = new[] { "Tier", "CapabilityPolicies", "RecycleIntervalSeconds" };
|
||||
|
||||
var actual = typeof(DriverResilienceOptions)
|
||||
.GetProperties()
|
||||
.Select(p => p.Name)
|
||||
.OrderBy(n => n)
|
||||
.ToArray();
|
||||
|
||||
actual.ShouldBe(expected.OrderBy(n => n).ToArray(),
|
||||
"DriverResilienceOptions must expose only pipeline-wired knobs — a new option needs a builder " +
|
||||
"strategy + a behavior test before it is added to the expected set (guards against the deleted " +
|
||||
"bulkhead 'parsed-but-unapplied' genre; see 01/U-6).");
|
||||
}
|
||||
|
||||
/// <summary>Verifies that tier defaults cover every capability.</summary>
|
||||
/// <param name="tier">The driver tier to test.</param>
|
||||
[Theory]
|
||||
|
||||
+79
@@ -325,6 +325,85 @@ public sealed class DriverResiliencePipelineBuilderTests
|
||||
e.Message.Contains("host-9"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 01/S-6 builder invariant: building a pipeline from a hostile, directly-constructed
|
||||
/// <see cref="CapabilityPolicy"/> (bypassing the parser) must NEVER throw Polly's
|
||||
/// <c>ValidationException</c>. Sweeps the full matrix of {int.MinValue, -1, 0, 1, int.MaxValue}
|
||||
/// across timeout/retry/breaker and asserts every built pipeline executes a trivial call.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Build_NeverThrows_ForHostileDirectOptions()
|
||||
{
|
||||
int[] hostile = { int.MinValue, -1, 0, 1, int.MaxValue };
|
||||
var host = 0;
|
||||
foreach (var t in hostile)
|
||||
foreach (var r in hostile)
|
||||
foreach (var b in hostile)
|
||||
{
|
||||
var options = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Read] = new(TimeoutSeconds: t, RetryCount: r, BreakerFailureThreshold: b),
|
||||
},
|
||||
};
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("hostile", $"h{host++}", DriverCapability.Read, options);
|
||||
|
||||
var result = await pipeline.ExecuteAsync(_ => ValueTask.FromResult(1), CancellationToken.None);
|
||||
result.ShouldBe(1, $"pipeline built from hostile (t={t}, r={r}, b={b}) must execute, not throw");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 01/S-7 = 03/S13: a stale-epoch re-cache must not be served to a new generation. Deterministic
|
||||
/// replay of the respawn race — the old child's late <c>GetOrCreate</c> re-caches under generation 1
|
||||
/// AFTER the invalidate; the new invoker (generation 2) with new options must build + serve a pipeline
|
||||
/// honoring the NEW options, not the poisoned old-generation entry. Generations make the interleaving
|
||||
/// expressible sequentially — no timing dependence.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task StaleGeneration_ReCache_IsNotServed_ToNewGeneration()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var optsA = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Read] = new(TimeoutSeconds: 5, RetryCount: 3, BreakerFailureThreshold: 0),
|
||||
},
|
||||
};
|
||||
var optsB = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Read] = new(TimeoutSeconds: 5, RetryCount: 0, BreakerFailureThreshold: 0),
|
||||
},
|
||||
};
|
||||
|
||||
// gen 1 warms the cache with the retrying optsA.
|
||||
builder.GetOrCreate("i", "h", DriverCapability.Read, optsA, optionsGeneration: 1);
|
||||
builder.Invalidate("i");
|
||||
// The lingering old child's late call re-caches optsA under its OWN generation 1.
|
||||
builder.GetOrCreate("i", "h", DriverCapability.Read, optsA, optionsGeneration: 1);
|
||||
|
||||
// The new invoker (gen 2, optsB) must get a pipeline honoring optsB (no retry), not the poisoned one.
|
||||
var newPipeline = builder.GetOrCreate("i", "h", DriverCapability.Read, optsB, optionsGeneration: 2);
|
||||
var attempts = 0;
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await newPipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
attempts++;
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
}));
|
||||
|
||||
attempts.ShouldBe(1, "the new generation must honor optsB (retryCount 0), not the stale optsA re-cache");
|
||||
}
|
||||
|
||||
/// <summary>Minimal in-memory <see cref="ILogger"/> capturing (level, formatted-message) pairs so the
|
||||
/// resilience logging contract can be asserted without a real logging provider.</summary>
|
||||
private sealed class CapturingLogger : ILogger
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
/// <summary>
|
||||
/// Load-bearing regression guard for 01/S-6 — an operator-authorable out-of-range
|
||||
/// ResilienceConfig value must NEVER brick a healthy driver. Before the R2-02 hardening pass,
|
||||
/// <c>"timeoutSeconds": 0</c> (→ <c>TimeSpan.Zero</c>, outside Polly's [10 ms, 24 h] window) and
|
||||
/// <c>"breakerFailureThreshold": 1</c> (below Polly's MinimumThroughput floor of 2) each made
|
||||
/// <see cref="DriverResiliencePipelineBuilder"/> throw <c>ValidationException</c> inside the
|
||||
/// memoizing <c>GetOrAdd</c> factory on EVERY capability call for that (instance, host,
|
||||
/// capability) — a permanent brick while the driver is healthy. The parser now clamps and the
|
||||
/// builder belt-and-braces clamps, so a pipeline built from a parsed hostile config executes.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class ResilienceConfigBrickTests
|
||||
{
|
||||
/// <summary>A parsed <c>timeoutSeconds:0</c> override must not brick a capability call.</summary>
|
||||
[Fact]
|
||||
public async Task ParsedConfig_WithZeroTimeout_DoesNotBrickCapabilityCalls()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"timeoutSeconds\":0}}}", out _);
|
||||
|
||||
var pipeline = new DriverResiliencePipelineBuilder()
|
||||
.GetOrCreate("i1", "h1", DriverCapability.Read, options);
|
||||
|
||||
var result = await pipeline.ExecuteAsync(_ => ValueTask.FromResult(42), CancellationToken.None);
|
||||
result.ShouldBe(42);
|
||||
}
|
||||
|
||||
/// <summary>A parsed <c>breakerFailureThreshold:1</c> override must not brick a capability call.</summary>
|
||||
[Fact]
|
||||
public async Task ParsedConfig_WithBreakerThresholdOne_DoesNotBrickCapabilityCalls()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.A, "{\"capabilityPolicies\":{\"Read\":{\"breakerFailureThreshold\":1}}}", out _);
|
||||
|
||||
var pipeline = new DriverResiliencePipelineBuilder()
|
||||
.GetOrCreate("i1", "h1", DriverCapability.Read, options);
|
||||
|
||||
var result = await pipeline.ExecuteAsync(_ => ValueTask.FromResult(42), CancellationToken.None);
|
||||
result.ShouldBe(42);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers;
|
||||
@@ -6,6 +7,59 @@ using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
public class ResilienceFormModelTests
|
||||
{
|
||||
// ---- 04/C-7: non-lossy round-trip ----
|
||||
|
||||
[Fact]
|
||||
public void Unknown_top_level_key_survives_round_trip()
|
||||
{
|
||||
// bulkheadMaxConcurrent doubles as the 01/U-6 continuity proof: the deleted knob's key survives.
|
||||
var json = """{"bulkheadMaxConcurrent":16,"capabilityPolicies":{"Read":{"timeoutSeconds":5}}}""";
|
||||
|
||||
var back = ResilienceFormModel.FromJson(json).ToJson();
|
||||
|
||||
back.ShouldNotBeNull();
|
||||
var obj = JsonNode.Parse(back)!.AsObject();
|
||||
obj["bulkheadMaxConcurrent"]!.GetValue<int>().ShouldBe(16);
|
||||
obj["capabilityPolicies"]!["Read"]!["timeoutSeconds"]!.GetValue<int>().ShouldBe(5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_capability_entry_survives_round_trip()
|
||||
{
|
||||
var json = """{"capabilityPolicies":{"Read":{"timeoutSeconds":5},"FutureCap":{"timeoutSeconds":9}}}""";
|
||||
|
||||
var back = ResilienceFormModel.FromJson(json).ToJson();
|
||||
|
||||
back.ShouldNotBeNull();
|
||||
var obj = JsonNode.Parse(back)!.AsObject();
|
||||
obj["capabilityPolicies"]!["FutureCap"]!["timeoutSeconds"]!.GetValue<int>().ShouldBe(9);
|
||||
obj["capabilityPolicies"]!["Read"]!["timeoutSeconds"]!.GetValue<int>().ShouldBe(5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_per_policy_field_survives_round_trip()
|
||||
{
|
||||
var json = """{"capabilityPolicies":{"Read":{"timeoutSeconds":5,"futureField":7}}}""";
|
||||
|
||||
var back = ResilienceFormModel.FromJson(json).ToJson();
|
||||
|
||||
back.ShouldNotBeNull();
|
||||
var read = JsonNode.Parse(back)!.AsObject()["capabilityPolicies"]!["Read"]!.AsObject();
|
||||
read["timeoutSeconds"]!.GetValue<int>().ShouldBe(5);
|
||||
read["futureField"]!.GetValue<int>().ShouldBe(7);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Malformed_json_sets_ParseFailed_and_ToJson_returns_original_text()
|
||||
{
|
||||
const string json = "{ not json";
|
||||
|
||||
var m = ResilienceFormModel.FromJson(json);
|
||||
|
||||
m.ParseFailed.ShouldBeTrue();
|
||||
m.ToJson().ShouldBe(json); // never overwrite what it couldn't read
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Blank_form_serializes_to_null()
|
||||
=> new ResilienceFormModel().ToJson().ShouldBeNull();
|
||||
@@ -13,7 +67,7 @@ public class ResilienceFormModelTests
|
||||
[Fact]
|
||||
public void Partial_override_round_trips()
|
||||
{
|
||||
var m = new ResilienceFormModel { BulkheadMaxConcurrent = 16 };
|
||||
var m = new ResilienceFormModel { RecycleIntervalSeconds = 3600 };
|
||||
m.Policies["Read"].TimeoutSeconds = 5;
|
||||
m.Policies["Read"].RetryCount = 5;
|
||||
|
||||
@@ -21,7 +75,7 @@ public class ResilienceFormModelTests
|
||||
json.ShouldNotBeNull();
|
||||
|
||||
var back = ResilienceFormModel.FromJson(json);
|
||||
back.BulkheadMaxConcurrent.ShouldBe(16);
|
||||
back.RecycleIntervalSeconds.ShouldBe(3600);
|
||||
back.Policies["Read"].TimeoutSeconds.ShouldBe(5);
|
||||
back.Policies["Write"].IsEmpty.ShouldBeTrue();
|
||||
}
|
||||
@@ -30,19 +84,48 @@ public class ResilienceFormModelTests
|
||||
public void Malformed_json_yields_empty_model()
|
||||
{
|
||||
var m = ResilienceFormModel.FromJson("{ not valid json");
|
||||
m.BulkheadMaxConcurrent.ShouldBeNull();
|
||||
m.RecycleIntervalSeconds.ShouldBeNull();
|
||||
m.Policies["Read"].IsEmpty.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_flags_out_of_range_values_with_capability_and_range()
|
||||
{
|
||||
var m = new ResilienceFormModel();
|
||||
m.Policies["Read"].TimeoutSeconds = 0; // < MinTimeoutSeconds
|
||||
m.Policies["Read"].RetryCount = -1; // < MinRetryCount
|
||||
m.Policies["Read"].BreakerFailureThreshold = 1; // below Polly MinimumThroughput floor
|
||||
|
||||
var msgs = m.Validate();
|
||||
|
||||
msgs.Count.ShouldBe(3);
|
||||
msgs.ShouldContain(s => s.Contains("Read") && s.Contains("timeout"));
|
||||
msgs.ShouldContain(s => s.Contains("Read") && s.Contains("retries"));
|
||||
msgs.ShouldContain(s => s.Contains("Read") && s.Contains("breaker"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_passes_in_range_and_blank()
|
||||
{
|
||||
var blank = new ResilienceFormModel();
|
||||
blank.Validate().ShouldBeEmpty();
|
||||
|
||||
var inRange = new ResilienceFormModel();
|
||||
inRange.Policies["Read"].TimeoutSeconds = 5;
|
||||
inRange.Policies["Read"].RetryCount = 3;
|
||||
inRange.Policies["Read"].BreakerFailureThreshold = 4;
|
||||
inRange.Policies["Write"].BreakerFailureThreshold = 0; // 0 = breaker off, legal
|
||||
inRange.Validate().ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Emitted_json_is_consumable_by_the_runtime_parser()
|
||||
{
|
||||
var m = new ResilienceFormModel { BulkheadMaxConcurrent = 16 };
|
||||
var m = new ResilienceFormModel();
|
||||
m.Policies["Read"].TimeoutSeconds = 7;
|
||||
|
||||
var opts = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.B, m.ToJson(), out var diag);
|
||||
diag.ShouldBeNull();
|
||||
opts.BulkheadMaxConcurrent.ShouldBe(16);
|
||||
opts.Resolve(DriverCapability.Read).TimeoutSeconds.ShouldBe(7);
|
||||
opts.Resolve(DriverCapability.Write).RetryCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
+5
@@ -75,6 +75,9 @@ public sealed class DriverInstanceActorResilienceWiringTests : RuntimeActorTestB
|
||||
reply!.Success.ShouldBeTrue();
|
||||
driver.Writes.Count.ShouldBeGreaterThanOrEqualTo(1); // the invoker's call-site actually ran
|
||||
invoker.Calls.ShouldContain(c => c.Capability == DriverCapability.Write && c.Host == "plc-7");
|
||||
// 03/S12: the dispatch site must default writes to NON-idempotent so the invoker's no-retry arm is
|
||||
// authoritative — a command-shaped write (pulse, counter, supervisory) must never replay.
|
||||
invoker.WriteCalls.ShouldContain(c => c.Host == "plc-7" && c.IsIdempotent == false);
|
||||
}
|
||||
|
||||
/// <summary>Records every capability + resolved-host key routed through it, then delegates to the
|
||||
@@ -83,6 +86,7 @@ public sealed class DriverInstanceActorResilienceWiringTests : RuntimeActorTestB
|
||||
private sealed class RecordingInvoker : IDriverCapabilityInvoker
|
||||
{
|
||||
public ConcurrentQueue<(DriverCapability Capability, string Host)> Calls { get; } = new();
|
||||
public ConcurrentQueue<(string Host, bool IsIdempotent)> WriteCalls { get; } = new();
|
||||
|
||||
public ValueTask<TResult> ExecuteAsync<TResult>(
|
||||
DriverCapability capability, string hostName,
|
||||
@@ -105,6 +109,7 @@ public sealed class DriverInstanceActorResilienceWiringTests : RuntimeActorTestB
|
||||
Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken)
|
||||
{
|
||||
Calls.Enqueue((DriverCapability.Write, hostName));
|
||||
WriteCalls.Enqueue((hostName, isIdempotent));
|
||||
return callSite(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user