Joseph Doherty
324 lines
13 KiB
C#
324 lines
13 KiB
C#
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
|
|
|
[Trait("Category", "Unit")]
|
|
public sealed class CapabilityInvokerTests
|
|
{
|
|
private static CapabilityInvoker MakeInvoker(
|
|
DriverResiliencePipelineBuilder builder,
|
|
DriverResilienceOptions options) =>
|
|
new(builder, "drv-test", () => options);
|
|
|
|
/// <summary>Verifies that the capability invoker returns the value from the call site.</summary>
|
|
[Fact]
|
|
public async Task Read_ReturnsValue_FromCallSite()
|
|
{
|
|
var invoker = MakeInvoker(new DriverResiliencePipelineBuilder(), new DriverResilienceOptions { Tier = DriverTier.A });
|
|
|
|
var result = await invoker.ExecuteAsync(
|
|
DriverCapability.Read,
|
|
"host-1",
|
|
_ => ValueTask.FromResult(42),
|
|
CancellationToken.None);
|
|
|
|
result.ShouldBe(42);
|
|
}
|
|
|
|
/// <summary>Verifies that the capability invoker retries on transient failures.</summary>
|
|
[Fact]
|
|
public async Task Read_Retries_OnTransientFailure()
|
|
{
|
|
var invoker = MakeInvoker(new DriverResiliencePipelineBuilder(), new DriverResilienceOptions { Tier = DriverTier.A });
|
|
var attempts = 0;
|
|
|
|
var result = await invoker.ExecuteAsync(
|
|
DriverCapability.Read,
|
|
"host-1",
|
|
async _ =>
|
|
{
|
|
attempts++;
|
|
if (attempts < 2) throw new InvalidOperationException("transient");
|
|
await Task.Yield();
|
|
return "ok";
|
|
},
|
|
CancellationToken.None);
|
|
|
|
result.ShouldBe("ok");
|
|
attempts.ShouldBe(2);
|
|
}
|
|
|
|
/// <summary>Verifies that non-idempotent writes do not retry even when the policy has retries configured.</summary>
|
|
[Fact]
|
|
public async Task Write_NonIdempotent_DoesNotRetry_EvenWhenPolicyHasRetries()
|
|
{
|
|
var options = new DriverResilienceOptions
|
|
{
|
|
Tier = DriverTier.A,
|
|
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
|
{
|
|
[DriverCapability.Write] = new(TimeoutSeconds: 2, RetryCount: 3, BreakerFailureThreshold: 5),
|
|
},
|
|
};
|
|
var invoker = MakeInvoker(new DriverResiliencePipelineBuilder(), options);
|
|
var attempts = 0;
|
|
|
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
|
await invoker.ExecuteWriteAsync(
|
|
"host-1",
|
|
isIdempotent: false,
|
|
async _ =>
|
|
{
|
|
attempts++;
|
|
await Task.Yield();
|
|
throw new InvalidOperationException("boom");
|
|
#pragma warning disable CS0162
|
|
return 0;
|
|
#pragma warning restore CS0162
|
|
},
|
|
CancellationToken.None));
|
|
|
|
attempts.ShouldBe(1, "non-idempotent write must never replay");
|
|
}
|
|
|
|
/// <summary>Verifies that idempotent writes retry when the policy has retries configured.</summary>
|
|
[Fact]
|
|
public async Task Write_Idempotent_Retries_WhenPolicyHasRetries()
|
|
{
|
|
var options = new DriverResilienceOptions
|
|
{
|
|
Tier = DriverTier.A,
|
|
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
|
{
|
|
[DriverCapability.Write] = new(TimeoutSeconds: 2, RetryCount: 3, BreakerFailureThreshold: 5),
|
|
},
|
|
};
|
|
var invoker = MakeInvoker(new DriverResiliencePipelineBuilder(), options);
|
|
var attempts = 0;
|
|
|
|
var result = await invoker.ExecuteWriteAsync(
|
|
"host-1",
|
|
isIdempotent: true,
|
|
async _ =>
|
|
{
|
|
attempts++;
|
|
if (attempts < 2) throw new InvalidOperationException("transient");
|
|
await Task.Yield();
|
|
return "ok";
|
|
},
|
|
CancellationToken.None);
|
|
|
|
result.ShouldBe("ok");
|
|
attempts.ShouldBe(2);
|
|
}
|
|
|
|
/// <summary>Verifies that writes do not retry when the policy has zero retries configured.</summary>
|
|
[Fact]
|
|
public async Task Write_Default_DoesNotRetry_WhenPolicyHasZeroRetries()
|
|
{
|
|
// Tier A Write default is RetryCount=0. Even isIdempotent=true shouldn't retry
|
|
// because the policy says not to.
|
|
var invoker = MakeInvoker(new DriverResiliencePipelineBuilder(), new DriverResilienceOptions { Tier = DriverTier.A });
|
|
var attempts = 0;
|
|
|
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
|
await invoker.ExecuteWriteAsync(
|
|
"host-1",
|
|
isIdempotent: true,
|
|
async _ =>
|
|
{
|
|
attempts++;
|
|
await Task.Yield();
|
|
throw new InvalidOperationException("boom");
|
|
#pragma warning disable CS0162
|
|
return 0;
|
|
#pragma warning restore CS0162
|
|
},
|
|
CancellationToken.None));
|
|
|
|
attempts.ShouldBe(1, "tier-A default for Write is RetryCount=0");
|
|
}
|
|
|
|
/// <summary>Verifies that different hosts are honored independently in the resilience pipeline.</summary>
|
|
[Fact]
|
|
public async Task Execute_HonorsDifferentHosts_Independently()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var invoker = MakeInvoker(builder, new DriverResilienceOptions { Tier = DriverTier.A });
|
|
|
|
await invoker.ExecuteAsync(DriverCapability.Read, "host-a", _ => ValueTask.FromResult(1), CancellationToken.None);
|
|
await invoker.ExecuteAsync(DriverCapability.Read, "host-b", _ => ValueTask.FromResult(2), CancellationToken.None);
|
|
|
|
builder.CachedPipelineCount.ShouldBe(2);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Core-009 regression: ExecuteWriteAsync's non-idempotent branch must snapshot
|
|
/// <c>_optionsAccessor</c> exactly once per call. Calling it multiple times allocates
|
|
/// redundant options objects on the per-write hot path and creates a consistency hazard
|
|
/// where an Admin edit mid-call could observe two different snapshots.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task ExecuteWriteAsync_NonIdempotent_Snapshots_Options_Once_Per_Call()
|
|
{
|
|
var options = new DriverResilienceOptions
|
|
{
|
|
Tier = DriverTier.A,
|
|
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
|
{
|
|
[DriverCapability.Write] = new(TimeoutSeconds: 2, RetryCount: 3, BreakerFailureThreshold: 5),
|
|
},
|
|
};
|
|
var accessorCalls = 0;
|
|
var invoker = new CapabilityInvoker(
|
|
new DriverResiliencePipelineBuilder(),
|
|
"drv-test",
|
|
() => { Interlocked.Increment(ref accessorCalls); return options; });
|
|
|
|
await invoker.ExecuteWriteAsync(
|
|
"host-1",
|
|
isIdempotent: false,
|
|
_ => ValueTask.FromResult(0),
|
|
CancellationToken.None);
|
|
|
|
accessorCalls.ShouldBe(1,
|
|
"ExecuteWriteAsync's non-idempotent branch must capture the options snapshot exactly once per call");
|
|
}
|
|
|
|
/// <summary>
|
|
/// S-8 residual: the non-idempotent write arm must record tracker in-flight accounting (start +
|
|
/// complete) on the caller's host, exactly like the idempotent arms — the previous arm skipped it.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task NonIdempotentWrite_RecordsTrackerStartAndComplete()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
var invoker = new CapabilityInvoker(
|
|
new DriverResiliencePipelineBuilder(),
|
|
"drv-test",
|
|
() => new DriverResilienceOptions { Tier = DriverTier.A },
|
|
statusTracker: tracker);
|
|
|
|
await invoker.ExecuteWriteAsync(
|
|
"host-1",
|
|
isIdempotent: false,
|
|
_ => ValueTask.FromResult(0),
|
|
CancellationToken.None);
|
|
|
|
var snap = tracker.TryGet("drv-test", "host-1");
|
|
snap.ShouldNotBeNull("the non-idempotent write arm must record tracker accounting on the caller's host");
|
|
snap!.CurrentInFlight.ShouldBe(0, "in-flight must return to 0 after the write completes");
|
|
}
|
|
|
|
/// <summary>
|
|
/// 01/P-4: the non-idempotent write arm must build the no-retry options snapshot ONCE per invoker
|
|
/// lifetime (not per call). Two writes to the same host reuse the cached snapshot — the options
|
|
/// accessor is invoked at most once and only one pipeline is cached.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task NonIdempotentWrite_ReusesCachedNoRetryOptions()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var accessorCalls = 0;
|
|
var invoker = new CapabilityInvoker(
|
|
builder,
|
|
"drv-test",
|
|
() => { Interlocked.Increment(ref accessorCalls); return new DriverResilienceOptions { Tier = DriverTier.A }; });
|
|
|
|
await invoker.ExecuteWriteAsync("host-1", isIdempotent: false, _ => ValueTask.FromResult(0), CancellationToken.None);
|
|
await invoker.ExecuteWriteAsync("host-1", isIdempotent: false, _ => ValueTask.FromResult(0), CancellationToken.None);
|
|
|
|
accessorCalls.ShouldBeLessThanOrEqualTo(1, "the no-retry snapshot must be cached across calls");
|
|
builder.CachedPipelineCount.ShouldBe(1, "two writes to the same host share one cached no-retry pipeline");
|
|
}
|
|
|
|
/// <summary>
|
|
/// R2-10: a successful capability call must reset <c>ConsecutiveFailures</c> so the counter
|
|
/// is a true "consecutive" gauge — nonzero only during an active retry storm, not stuck at the
|
|
/// last blip's count forever. Previously only the breaker's <c>OnClosed</c> reset it, so a
|
|
/// transient 2-retry blip that never opened the breaker read "2 failures" indefinitely.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task ExecuteAsync_OnSuccess_ResetsConsecutiveFailures()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
tracker.RecordFailure("drv-test", "host-1", DateTime.UtcNow);
|
|
tracker.RecordFailure("drv-test", "host-1", DateTime.UtcNow);
|
|
tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(2, "seeded failures");
|
|
|
|
var invoker = new CapabilityInvoker(
|
|
new DriverResiliencePipelineBuilder(),
|
|
"drv-test",
|
|
() => new DriverResilienceOptions { Tier = DriverTier.A },
|
|
statusTracker: tracker);
|
|
|
|
await invoker.ExecuteAsync(
|
|
DriverCapability.Read,
|
|
"host-1",
|
|
_ => ValueTask.FromResult(1),
|
|
CancellationToken.None);
|
|
|
|
tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(0,
|
|
"a successful capability call must clear the consecutive-failure counter");
|
|
}
|
|
|
|
/// <summary>
|
|
/// R2-10 companion: a successful non-idempotent write must also reset the consecutive-failure
|
|
/// counter on the caller's host (the write arm has its own success path).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task ExecuteWriteAsync_NonIdempotent_OnSuccess_ResetsConsecutiveFailures()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
tracker.RecordFailure("drv-test", "host-1", DateTime.UtcNow);
|
|
tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(1, "seeded failure");
|
|
|
|
var invoker = new CapabilityInvoker(
|
|
new DriverResiliencePipelineBuilder(),
|
|
"drv-test",
|
|
() => new DriverResilienceOptions { Tier = DriverTier.A },
|
|
statusTracker: tracker);
|
|
|
|
await invoker.ExecuteWriteAsync(
|
|
"host-1",
|
|
isIdempotent: false,
|
|
_ => ValueTask.FromResult(0),
|
|
CancellationToken.None);
|
|
|
|
tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(0,
|
|
"a successful non-idempotent write must clear the consecutive-failure counter");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Core-009 regression — companion consistency assertion: the non-idempotent branch must
|
|
/// not observe two different option snapshots even if the accessor's returned value changes
|
|
/// between calls (simulating an Admin edit landing mid-flight). With a single snapshot the
|
|
/// two derived values (<c>with</c> base + <c>Resolve(Write)</c>) come from the same options
|
|
/// instance.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task ExecuteWriteAsync_NonIdempotent_Uses_Consistent_Options_Snapshot()
|
|
{
|
|
var a = new DriverResilienceOptions { Tier = DriverTier.A };
|
|
var b = new DriverResilienceOptions { Tier = DriverTier.B };
|
|
var alternating = new[] { a, b, a, b }.AsEnumerable().GetEnumerator();
|
|
var invoker = new CapabilityInvoker(
|
|
new DriverResiliencePipelineBuilder(),
|
|
"drv-test",
|
|
() => { alternating.MoveNext(); return alternating.Current; });
|
|
|
|
// If options is read twice, the with-expression and Resolve(Write) come from
|
|
// different tier tables (A then B) — the resulting one-entry dictionary is
|
|
// inconsistent with the snapshot used for the rest of the options. Single-snapshot
|
|
// semantics guarantee the call sees a coherent view.
|
|
await Should.NotThrowAsync(async () => await invoker.ExecuteWriteAsync(
|
|
"host-1",
|
|
isIdempotent: false,
|
|
_ => ValueTask.FromResult(0),
|
|
CancellationToken.None));
|
|
}
|
|
}
|