Joseph Doherty
376 lines
14 KiB
C#
376 lines
14 KiB
C#
using Microsoft.Extensions.Logging;
|
|
using Polly.CircuitBreaker;
|
|
using Polly.Timeout;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
|
|
|
[Trait("Category", "Unit")]
|
|
public sealed class DriverResiliencePipelineBuilderTests
|
|
{
|
|
private static readonly DriverResilienceOptions TierAOptions = new() { Tier = DriverTier.A };
|
|
|
|
/// <summary>Verifies that read operations retry transient failures.</summary>
|
|
[Fact]
|
|
public async Task Read_Retries_Transient_Failures()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions);
|
|
var attempts = 0;
|
|
|
|
await pipeline.ExecuteAsync(async _ =>
|
|
{
|
|
attempts++;
|
|
if (attempts < 3) throw new InvalidOperationException("transient");
|
|
await Task.Yield();
|
|
});
|
|
|
|
attempts.ShouldBe(3);
|
|
}
|
|
|
|
/// <summary>Verifies that write operations do not retry on failure.</summary>
|
|
[Fact]
|
|
public async Task Write_DoesNotRetry_OnFailure()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions);
|
|
var attempts = 0;
|
|
|
|
var ex = await Should.ThrowAsync<InvalidOperationException>(async () =>
|
|
{
|
|
await pipeline.ExecuteAsync(async _ =>
|
|
{
|
|
attempts++;
|
|
await Task.Yield();
|
|
throw new InvalidOperationException("boom");
|
|
});
|
|
});
|
|
|
|
attempts.ShouldBe(1);
|
|
ex.Message.ShouldBe("boom");
|
|
}
|
|
|
|
/// <summary>Verifies that alarm acknowledge operations do not retry on failure.</summary>
|
|
[Fact]
|
|
public async Task AlarmAcknowledge_DoesNotRetry_OnFailure()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.AlarmAcknowledge, TierAOptions);
|
|
var attempts = 0;
|
|
|
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
|
{
|
|
await pipeline.ExecuteAsync(async _ =>
|
|
{
|
|
attempts++;
|
|
await Task.Yield();
|
|
throw new InvalidOperationException("boom");
|
|
});
|
|
});
|
|
|
|
attempts.ShouldBe(1);
|
|
}
|
|
|
|
/// <summary>Verifies that pipelines are isolated per host.</summary>
|
|
[Fact]
|
|
public void Pipeline_IsIsolated_PerHost()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var driverId = "drv-test";
|
|
|
|
var hostA = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
|
var hostB = builder.GetOrCreate(driverId, "host-b", DriverCapability.Read, TierAOptions);
|
|
|
|
hostA.ShouldNotBeSameAs(hostB);
|
|
builder.CachedPipelineCount.ShouldBe(2);
|
|
}
|
|
|
|
/// <summary>Verifies that pipelines are reused for the same driver, host, and capability triple.</summary>
|
|
[Fact]
|
|
public void Pipeline_IsReused_ForSameTriple()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var driverId = "drv-test";
|
|
|
|
var first = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
|
var second = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
|
|
|
first.ShouldBeSameAs(second);
|
|
builder.CachedPipelineCount.ShouldBe(1);
|
|
}
|
|
|
|
/// <summary>Verifies that pipelines are isolated per capability.</summary>
|
|
[Fact]
|
|
public void Pipeline_IsIsolated_PerCapability()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var driverId = "drv-test";
|
|
|
|
var read = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
|
var write = builder.GetOrCreate(driverId, "host-a", DriverCapability.Write, TierAOptions);
|
|
|
|
read.ShouldNotBeSameAs(write);
|
|
}
|
|
|
|
/// <summary>Verifies that a dead host does not open the breaker for a sibling host.</summary>
|
|
[Fact]
|
|
public async Task DeadHost_DoesNotOpenBreaker_ForSiblingHost()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var driverId = "drv-test";
|
|
|
|
var deadHost = builder.GetOrCreate(driverId, "dead-plc", DriverCapability.Read, TierAOptions);
|
|
var liveHost = builder.GetOrCreate(driverId, "live-plc", DriverCapability.Read, TierAOptions);
|
|
|
|
var threshold = TierAOptions.Resolve(DriverCapability.Read).BreakerFailureThreshold;
|
|
for (var i = 0; i < threshold + 5; i++)
|
|
{
|
|
await Should.ThrowAsync<Exception>(async () =>
|
|
await deadHost.ExecuteAsync(async _ =>
|
|
{
|
|
await Task.Yield();
|
|
throw new InvalidOperationException("dead plc");
|
|
}));
|
|
}
|
|
|
|
var liveAttempts = 0;
|
|
await liveHost.ExecuteAsync(async _ =>
|
|
{
|
|
liveAttempts++;
|
|
await Task.Yield();
|
|
});
|
|
|
|
liveAttempts.ShouldBe(1, "healthy sibling host must not be affected by dead peer");
|
|
}
|
|
|
|
/// <summary>Verifies that the circuit breaker opens after the failure threshold on tier A.</summary>
|
|
[Fact]
|
|
public async Task CircuitBreaker_Opens_AfterFailureThreshold_OnTierA()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions);
|
|
|
|
var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold;
|
|
for (var i = 0; i < threshold; i++)
|
|
{
|
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
|
await pipeline.ExecuteAsync(async _ =>
|
|
{
|
|
await Task.Yield();
|
|
throw new InvalidOperationException("boom");
|
|
}));
|
|
}
|
|
|
|
await Should.ThrowAsync<BrokenCircuitException>(async () =>
|
|
await pipeline.ExecuteAsync(async _ =>
|
|
{
|
|
await Task.Yield();
|
|
}));
|
|
}
|
|
|
|
/// <summary>Verifies that timeout cancels slow operations.</summary>
|
|
[Fact]
|
|
public async Task Timeout_Cancels_SlowOperation()
|
|
{
|
|
var tierAWithShortTimeout = new DriverResilienceOptions
|
|
{
|
|
Tier = DriverTier.A,
|
|
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
|
{
|
|
[DriverCapability.Read] = new(TimeoutSeconds: 1, RetryCount: 0, BreakerFailureThreshold: 5),
|
|
},
|
|
};
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, tierAWithShortTimeout);
|
|
|
|
await Should.ThrowAsync<TimeoutRejectedException>(async () =>
|
|
await pipeline.ExecuteAsync(async ct =>
|
|
{
|
|
await Task.Delay(TimeSpan.FromSeconds(5), ct);
|
|
}));
|
|
}
|
|
|
|
/// <summary>Verifies that invalidate removes only the matching instance.</summary>
|
|
[Fact]
|
|
public void Invalidate_Removes_OnlyMatchingInstance()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var keepId = "drv-keep";
|
|
var dropId = "drv-drop";
|
|
|
|
builder.GetOrCreate(keepId, "h", DriverCapability.Read, TierAOptions);
|
|
builder.GetOrCreate(keepId, "h", DriverCapability.Write, TierAOptions);
|
|
builder.GetOrCreate(dropId, "h", DriverCapability.Read, TierAOptions);
|
|
|
|
var removed = builder.Invalidate(dropId);
|
|
|
|
removed.ShouldBe(1);
|
|
builder.CachedPipelineCount.ShouldBe(2);
|
|
}
|
|
|
|
/// <summary>Verifies that cancellation is not retried.</summary>
|
|
[Fact]
|
|
public async Task Cancellation_IsNot_Retried()
|
|
{
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions);
|
|
var attempts = 0;
|
|
using var cts = new CancellationTokenSource();
|
|
cts.Cancel();
|
|
|
|
await Should.ThrowAsync<OperationCanceledException>(async () =>
|
|
await pipeline.ExecuteAsync(async ct =>
|
|
{
|
|
attempts++;
|
|
ct.ThrowIfCancellationRequested();
|
|
await Task.Yield();
|
|
}, cts.Token));
|
|
|
|
attempts.ShouldBeLessThanOrEqualTo(1);
|
|
}
|
|
|
|
/// <summary>Verifies that the tracker records failure on every retry.</summary>
|
|
[Fact]
|
|
public async Task Tracker_RecordsFailure_OnEveryRetry()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
|
var pipeline = builder.GetOrCreate("drv-trk", "host-x", DriverCapability.Read, TierAOptions);
|
|
|
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
|
await pipeline.ExecuteAsync(async _ =>
|
|
{
|
|
await Task.Yield();
|
|
throw new InvalidOperationException("always fails");
|
|
}));
|
|
|
|
var snap = tracker.TryGet("drv-trk", "host-x");
|
|
snap.ShouldNotBeNull();
|
|
var retryCount = TierAOptions.Resolve(DriverCapability.Read).RetryCount;
|
|
snap!.ConsecutiveFailures.ShouldBe(retryCount);
|
|
}
|
|
|
|
/// <summary>Verifies that the tracker stamps the breaker open when it trips.</summary>
|
|
[Fact]
|
|
public async Task Tracker_StampsBreakerOpen_WhenBreakerTrips()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
|
var pipeline = builder.GetOrCreate("drv-trk", "host-b", DriverCapability.Write, TierAOptions);
|
|
|
|
var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold;
|
|
for (var i = 0; i < threshold; i++)
|
|
{
|
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
|
await pipeline.ExecuteAsync(async _ =>
|
|
{
|
|
await Task.Yield();
|
|
throw new InvalidOperationException("boom");
|
|
}));
|
|
}
|
|
|
|
var snap = tracker.TryGet("drv-trk", "host-b");
|
|
snap.ShouldNotBeNull();
|
|
snap!.LastBreakerOpenUtc.ShouldNotBeNull();
|
|
}
|
|
|
|
/// <summary>Verifies that the tracker isolates counters per host.</summary>
|
|
[Fact]
|
|
public async Task Tracker_IsolatesCounters_PerHost()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
|
var dead = builder.GetOrCreate("drv-trk", "dead", DriverCapability.Read, TierAOptions);
|
|
var live = builder.GetOrCreate("drv-trk", "live", DriverCapability.Read, TierAOptions);
|
|
|
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
|
await dead.ExecuteAsync(async _ =>
|
|
{
|
|
await Task.Yield();
|
|
throw new InvalidOperationException("dead");
|
|
}));
|
|
await live.ExecuteAsync(async _ => await Task.Yield());
|
|
|
|
tracker.TryGet("drv-trk", "dead")!.ConsecutiveFailures.ShouldBeGreaterThan(0);
|
|
tracker.TryGet("drv-trk", "live").ShouldBeNull();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Retry events are logged when a logger is supplied — this is the operator-facing observability
|
|
/// surface the arch-review #10 live-verify relies on (the Admin <c>/hosts</c> reader isn't built).
|
|
/// Without it the resilience pipeline runs silently and a live retry/breaker transition is invisible.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Retry_Is_Logged_When_Logger_Supplied()
|
|
{
|
|
var logger = new CapturingLogger();
|
|
var builder = new DriverResiliencePipelineBuilder(logger: logger);
|
|
var pipeline = builder.GetOrCreate("drv-log", "host-9", DriverCapability.Read, TierAOptions);
|
|
var attempts = 0;
|
|
|
|
await pipeline.ExecuteAsync(async _ =>
|
|
{
|
|
attempts++;
|
|
if (attempts < 3) throw new InvalidOperationException("transient");
|
|
await Task.Yield();
|
|
});
|
|
|
|
attempts.ShouldBe(3);
|
|
logger.Entries.ShouldContain(e =>
|
|
e.Level == LogLevel.Warning &&
|
|
e.Message.Contains("resilience retry") &&
|
|
e.Message.Contains("host-9"));
|
|
}
|
|
|
|
/// <summary>
|
|
/// 01/S-6 builder invariant: building a pipeline from a hostile, directly-constructed
|
|
/// <see cref="CapabilityPolicy"/> (bypassing the parser) must NEVER throw Polly's
|
|
/// <c>ValidationException</c>. Sweeps the full matrix of {int.MinValue, -1, 0, 1, int.MaxValue}
|
|
/// across timeout/retry/breaker and asserts every built pipeline executes a trivial call.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Build_NeverThrows_ForHostileDirectOptions()
|
|
{
|
|
int[] hostile = { int.MinValue, -1, 0, 1, int.MaxValue };
|
|
var host = 0;
|
|
foreach (var t in hostile)
|
|
foreach (var r in hostile)
|
|
foreach (var b in hostile)
|
|
{
|
|
var options = new DriverResilienceOptions
|
|
{
|
|
Tier = DriverTier.A,
|
|
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
|
{
|
|
[DriverCapability.Read] = new(TimeoutSeconds: t, RetryCount: r, BreakerFailureThreshold: b),
|
|
},
|
|
};
|
|
var builder = new DriverResiliencePipelineBuilder();
|
|
var pipeline = builder.GetOrCreate("hostile", $"h{host++}", DriverCapability.Read, options);
|
|
|
|
var result = await pipeline.ExecuteAsync(_ => ValueTask.FromResult(1), CancellationToken.None);
|
|
result.ShouldBe(1, $"pipeline built from hostile (t={t}, r={r}, b={b}) must execute, not throw");
|
|
}
|
|
}
|
|
|
|
/// <summary>Minimal in-memory <see cref="ILogger"/> capturing (level, formatted-message) pairs so the
|
|
/// resilience logging contract can be asserted without a real logging provider.</summary>
|
|
private sealed class CapturingLogger : ILogger
|
|
{
|
|
public List<(LogLevel Level, string Message)> Entries { get; } = new();
|
|
public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullScope.Instance;
|
|
public bool IsEnabled(LogLevel logLevel) => true;
|
|
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
|
|
Func<TState, Exception?, string> formatter) => Entries.Add((logLevel, formatter(state, exception)));
|
|
|
|
private sealed class NullScope : IDisposable
|
|
{
|
|
public static readonly NullScope Instance = new();
|
|
public void Dispose() { }
|
|
}
|
|
}
|
|
}
|