chore: organize solution into module folders (Core/Server/Drivers/Client/Tooling)
Group all 69 projects into category subfolders under src/ and tests/ so the Rider Solution Explorer mirrors the module structure. Folders: Core, Server, Drivers (with a nested Driver CLIs subfolder), Client, Tooling. - Move every project folder on disk with git mv (history preserved as renames). - Recompute relative paths in 57 .csproj files: cross-category ProjectReferences, the lib/ HintPath+None refs in Driver.Historian.Wonderware, and the external mxaccessgw refs in Driver.Galaxy and its test project. - Rebuild ZB.MOM.WW.OtOpcUa.slnx with nested solution folders. - Re-prefix project paths in functional scripts (e2e, compliance, smoke SQL, integration, install). Build green (0 errors); unit tests pass. Docs left for a separate pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,285 @@
|
||||
using Polly.CircuitBreaker;
|
||||
using Polly.Timeout;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class DriverResiliencePipelineBuilderTests
|
||||
{
|
||||
private static readonly DriverResilienceOptions TierAOptions = new() { Tier = DriverTier.A };
|
||||
|
||||
[Fact]
|
||||
public async Task Read_Retries_Transient_Failures()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions);
|
||||
var attempts = 0;
|
||||
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
attempts++;
|
||||
if (attempts < 3) throw new InvalidOperationException("transient");
|
||||
await Task.Yield();
|
||||
});
|
||||
|
||||
attempts.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Write_DoesNotRetry_OnFailure()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions);
|
||||
var attempts = 0;
|
||||
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
{
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
attempts++;
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
});
|
||||
});
|
||||
|
||||
attempts.ShouldBe(1);
|
||||
ex.Message.ShouldBe("boom");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AlarmAcknowledge_DoesNotRetry_OnFailure()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.AlarmAcknowledge, TierAOptions);
|
||||
var attempts = 0;
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
{
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
attempts++;
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
});
|
||||
});
|
||||
|
||||
attempts.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pipeline_IsIsolated_PerHost()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var driverId = "drv-test";
|
||||
|
||||
var hostA = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
||||
var hostB = builder.GetOrCreate(driverId, "host-b", DriverCapability.Read, TierAOptions);
|
||||
|
||||
hostA.ShouldNotBeSameAs(hostB);
|
||||
builder.CachedPipelineCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pipeline_IsReused_ForSameTriple()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var driverId = "drv-test";
|
||||
|
||||
var first = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
||||
var second = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
||||
|
||||
first.ShouldBeSameAs(second);
|
||||
builder.CachedPipelineCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pipeline_IsIsolated_PerCapability()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var driverId = "drv-test";
|
||||
|
||||
var read = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
||||
var write = builder.GetOrCreate(driverId, "host-a", DriverCapability.Write, TierAOptions);
|
||||
|
||||
read.ShouldNotBeSameAs(write);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeadHost_DoesNotOpenBreaker_ForSiblingHost()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var driverId = "drv-test";
|
||||
|
||||
var deadHost = builder.GetOrCreate(driverId, "dead-plc", DriverCapability.Read, TierAOptions);
|
||||
var liveHost = builder.GetOrCreate(driverId, "live-plc", DriverCapability.Read, TierAOptions);
|
||||
|
||||
var threshold = TierAOptions.Resolve(DriverCapability.Read).BreakerFailureThreshold;
|
||||
for (var i = 0; i < threshold + 5; i++)
|
||||
{
|
||||
await Should.ThrowAsync<Exception>(async () =>
|
||||
await deadHost.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("dead plc");
|
||||
}));
|
||||
}
|
||||
|
||||
var liveAttempts = 0;
|
||||
await liveHost.ExecuteAsync(async _ =>
|
||||
{
|
||||
liveAttempts++;
|
||||
await Task.Yield();
|
||||
});
|
||||
|
||||
liveAttempts.ShouldBe(1, "healthy sibling host must not be affected by dead peer");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CircuitBreaker_Opens_AfterFailureThreshold_OnTierA()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions);
|
||||
|
||||
var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold;
|
||||
for (var i = 0; i < threshold; i++)
|
||||
{
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
}));
|
||||
}
|
||||
|
||||
await Should.ThrowAsync<BrokenCircuitException>(async () =>
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
}));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Timeout_Cancels_SlowOperation()
|
||||
{
|
||||
var tierAWithShortTimeout = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Read] = new(TimeoutSeconds: 1, RetryCount: 0, BreakerFailureThreshold: 5),
|
||||
},
|
||||
};
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, tierAWithShortTimeout);
|
||||
|
||||
await Should.ThrowAsync<TimeoutRejectedException>(async () =>
|
||||
await pipeline.ExecuteAsync(async ct =>
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), ct);
|
||||
}));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Invalidate_Removes_OnlyMatchingInstance()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var keepId = "drv-keep";
|
||||
var dropId = "drv-drop";
|
||||
|
||||
builder.GetOrCreate(keepId, "h", DriverCapability.Read, TierAOptions);
|
||||
builder.GetOrCreate(keepId, "h", DriverCapability.Write, TierAOptions);
|
||||
builder.GetOrCreate(dropId, "h", DriverCapability.Read, TierAOptions);
|
||||
|
||||
var removed = builder.Invalidate(dropId);
|
||||
|
||||
removed.ShouldBe(1);
|
||||
builder.CachedPipelineCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cancellation_IsNot_Retried()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions);
|
||||
var attempts = 0;
|
||||
using var cts = new CancellationTokenSource();
|
||||
cts.Cancel();
|
||||
|
||||
await Should.ThrowAsync<OperationCanceledException>(async () =>
|
||||
await pipeline.ExecuteAsync(async ct =>
|
||||
{
|
||||
attempts++;
|
||||
ct.ThrowIfCancellationRequested();
|
||||
await Task.Yield();
|
||||
}, cts.Token));
|
||||
|
||||
attempts.ShouldBeLessThanOrEqualTo(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tracker_RecordsFailure_OnEveryRetry()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
||||
var pipeline = builder.GetOrCreate("drv-trk", "host-x", DriverCapability.Read, TierAOptions);
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("always fails");
|
||||
}));
|
||||
|
||||
var snap = tracker.TryGet("drv-trk", "host-x");
|
||||
snap.ShouldNotBeNull();
|
||||
var retryCount = TierAOptions.Resolve(DriverCapability.Read).RetryCount;
|
||||
snap!.ConsecutiveFailures.ShouldBe(retryCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tracker_StampsBreakerOpen_WhenBreakerTrips()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
||||
var pipeline = builder.GetOrCreate("drv-trk", "host-b", DriverCapability.Write, TierAOptions);
|
||||
|
||||
var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold;
|
||||
for (var i = 0; i < threshold; i++)
|
||||
{
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
}));
|
||||
}
|
||||
|
||||
var snap = tracker.TryGet("drv-trk", "host-b");
|
||||
snap.ShouldNotBeNull();
|
||||
snap!.LastBreakerOpenUtc.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tracker_IsolatesCounters_PerHost()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
||||
var dead = builder.GetOrCreate("drv-trk", "dead", DriverCapability.Read, TierAOptions);
|
||||
var live = builder.GetOrCreate("drv-trk", "live", DriverCapability.Read, TierAOptions);
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await dead.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("dead");
|
||||
}));
|
||||
await live.ExecuteAsync(async _ => await Task.Yield());
|
||||
|
||||
tracker.TryGet("drv-trk", "dead")!.ConsecutiveFailures.ShouldBeGreaterThan(0);
|
||||
tracker.TryGet("drv-trk", "live").ShouldBeNull();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user