chore: organize solution into module folders (Core/Server/Drivers/Client/Tooling)
Group all 69 projects into category subfolders under src/ and tests/ so the Rider Solution Explorer mirrors the module structure. Folders: Core, Server, Drivers (with a nested Driver CLIs subfolder), Client, Tooling. - Move every project folder on disk with git mv (history preserved as renames). - Recompute relative paths in 57 .csproj files: cross-category ProjectReferences, the lib/ HintPath+None refs in Driver.Historian.Wonderware, and the external mxaccessgw refs in Driver.Galaxy and its test project. - Rebuild ZB.MOM.WW.OtOpcUa.slnx with nested solution folders. - Re-prefix project paths in functional scripts (e2e, compliance, smoke SQL, integration, install). Build green (0 errors); unit tests pass. Docs left for a separate pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,127 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class AlarmSurfaceInvokerTests
|
||||
{
|
||||
private static readonly DriverResilienceOptions TierAOptions = new() { Tier = DriverTier.A };
|
||||
|
||||
[Fact]
|
||||
public async Task SubscribeAsync_EmptyList_ReturnsEmpty_WithoutDriverCall()
|
||||
{
|
||||
var driver = new FakeAlarmSource();
|
||||
var surface = NewSurface(driver, defaultHost: "h");
|
||||
|
||||
var handles = await surface.SubscribeAsync([], CancellationToken.None);
|
||||
|
||||
handles.Count.ShouldBe(0);
|
||||
driver.SubscribeCallCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SubscribeAsync_SingleHost_RoutesThroughDefaultHost()
|
||||
{
|
||||
var driver = new FakeAlarmSource();
|
||||
var surface = NewSurface(driver, defaultHost: "h1");
|
||||
|
||||
var handles = await surface.SubscribeAsync(["src-1", "src-2"], CancellationToken.None);
|
||||
|
||||
handles.Count.ShouldBe(1);
|
||||
driver.SubscribeCallCount.ShouldBe(1);
|
||||
driver.LastSubscribedIds.ShouldBe(["src-1", "src-2"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SubscribeAsync_MultiHost_FansOutByResolvedHost()
|
||||
{
|
||||
var driver = new FakeAlarmSource();
|
||||
var resolver = new StubResolver(new Dictionary<string, string>
|
||||
{
|
||||
["src-1"] = "plc-a",
|
||||
["src-2"] = "plc-b",
|
||||
["src-3"] = "plc-a",
|
||||
});
|
||||
var surface = NewSurface(driver, defaultHost: "default-ignored", resolver: resolver);
|
||||
|
||||
var handles = await surface.SubscribeAsync(["src-1", "src-2", "src-3"], CancellationToken.None);
|
||||
|
||||
handles.Count.ShouldBe(2); // one per distinct host
|
||||
driver.SubscribeCallCount.ShouldBe(2); // one driver call per host
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AcknowledgeAsync_DoesNotRetry_OnFailure()
|
||||
{
|
||||
var driver = new FakeAlarmSource { AcknowledgeShouldThrow = true };
|
||||
var surface = NewSurface(driver, defaultHost: "h1");
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(() =>
|
||||
surface.AcknowledgeAsync([new AlarmAcknowledgeRequest("s", "c", null)], CancellationToken.None));
|
||||
|
||||
driver.AcknowledgeCallCount.ShouldBe(1, "AlarmAcknowledge must not retry — decision #143");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SubscribeAsync_Retries_Transient_Failures()
|
||||
{
|
||||
var driver = new FakeAlarmSource { SubscribeFailuresBeforeSuccess = 2 };
|
||||
var surface = NewSurface(driver, defaultHost: "h1");
|
||||
|
||||
await surface.SubscribeAsync(["src"], CancellationToken.None);
|
||||
|
||||
driver.SubscribeCallCount.ShouldBe(3, "AlarmSubscribe retries by default — decision #143");
|
||||
}
|
||||
|
||||
private static AlarmSurfaceInvoker NewSurface(
|
||||
IAlarmSource driver,
|
||||
string defaultHost,
|
||||
IPerCallHostResolver? resolver = null)
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var invoker = new CapabilityInvoker(builder, "drv-1", () => TierAOptions);
|
||||
return new AlarmSurfaceInvoker(invoker, driver, defaultHost, resolver);
|
||||
}
|
||||
|
||||
private sealed class FakeAlarmSource : IAlarmSource
|
||||
{
|
||||
public int SubscribeCallCount { get; private set; }
|
||||
public int AcknowledgeCallCount { get; private set; }
|
||||
public int SubscribeFailuresBeforeSuccess { get; set; }
|
||||
public bool AcknowledgeShouldThrow { get; set; }
|
||||
public IReadOnlyList<string> LastSubscribedIds { get; private set; } = [];
|
||||
|
||||
public Task<IAlarmSubscriptionHandle> SubscribeAlarmsAsync(
|
||||
IReadOnlyList<string> sourceNodeIds, CancellationToken cancellationToken)
|
||||
{
|
||||
SubscribeCallCount++;
|
||||
LastSubscribedIds = sourceNodeIds;
|
||||
if (SubscribeCallCount <= SubscribeFailuresBeforeSuccess)
|
||||
throw new InvalidOperationException("transient");
|
||||
return Task.FromResult<IAlarmSubscriptionHandle>(new StubHandle($"h-{SubscribeCallCount}"));
|
||||
}
|
||||
|
||||
public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task AcknowledgeAsync(
|
||||
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken)
|
||||
{
|
||||
AcknowledgeCallCount++;
|
||||
if (AcknowledgeShouldThrow) throw new InvalidOperationException("ack boom");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public event EventHandler<AlarmEventArgs>? OnAlarmEvent { add { } remove { } }
|
||||
}
|
||||
|
||||
private sealed record StubHandle(string DiagnosticId) : IAlarmSubscriptionHandle;
|
||||
|
||||
private sealed class StubResolver(Dictionary<string, string> map) : IPerCallHostResolver
|
||||
{
|
||||
public string ResolveHost(string fullReference) => map[fullReference];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class CapabilityInvokerTests
|
||||
{
|
||||
private static CapabilityInvoker MakeInvoker(
|
||||
DriverResiliencePipelineBuilder builder,
|
||||
DriverResilienceOptions options) =>
|
||||
new(builder, "drv-test", () => options);
|
||||
|
||||
[Fact]
|
||||
public async Task Read_ReturnsValue_FromCallSite()
|
||||
{
|
||||
var invoker = MakeInvoker(new DriverResiliencePipelineBuilder(), new DriverResilienceOptions { Tier = DriverTier.A });
|
||||
|
||||
var result = await invoker.ExecuteAsync(
|
||||
DriverCapability.Read,
|
||||
"host-1",
|
||||
_ => ValueTask.FromResult(42),
|
||||
CancellationToken.None);
|
||||
|
||||
result.ShouldBe(42);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Read_Retries_OnTransientFailure()
|
||||
{
|
||||
var invoker = MakeInvoker(new DriverResiliencePipelineBuilder(), new DriverResilienceOptions { Tier = DriverTier.A });
|
||||
var attempts = 0;
|
||||
|
||||
var result = await invoker.ExecuteAsync(
|
||||
DriverCapability.Read,
|
||||
"host-1",
|
||||
async _ =>
|
||||
{
|
||||
attempts++;
|
||||
if (attempts < 2) throw new InvalidOperationException("transient");
|
||||
await Task.Yield();
|
||||
return "ok";
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
result.ShouldBe("ok");
|
||||
attempts.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Write_NonIdempotent_DoesNotRetry_EvenWhenPolicyHasRetries()
|
||||
{
|
||||
var options = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Write] = new(TimeoutSeconds: 2, RetryCount: 3, BreakerFailureThreshold: 5),
|
||||
},
|
||||
};
|
||||
var invoker = MakeInvoker(new DriverResiliencePipelineBuilder(), options);
|
||||
var attempts = 0;
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await invoker.ExecuteWriteAsync(
|
||||
"host-1",
|
||||
isIdempotent: false,
|
||||
async _ =>
|
||||
{
|
||||
attempts++;
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
#pragma warning disable CS0162
|
||||
return 0;
|
||||
#pragma warning restore CS0162
|
||||
},
|
||||
CancellationToken.None));
|
||||
|
||||
attempts.ShouldBe(1, "non-idempotent write must never replay");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Write_Idempotent_Retries_WhenPolicyHasRetries()
|
||||
{
|
||||
var options = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Write] = new(TimeoutSeconds: 2, RetryCount: 3, BreakerFailureThreshold: 5),
|
||||
},
|
||||
};
|
||||
var invoker = MakeInvoker(new DriverResiliencePipelineBuilder(), options);
|
||||
var attempts = 0;
|
||||
|
||||
var result = await invoker.ExecuteWriteAsync(
|
||||
"host-1",
|
||||
isIdempotent: true,
|
||||
async _ =>
|
||||
{
|
||||
attempts++;
|
||||
if (attempts < 2) throw new InvalidOperationException("transient");
|
||||
await Task.Yield();
|
||||
return "ok";
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
result.ShouldBe("ok");
|
||||
attempts.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Write_Default_DoesNotRetry_WhenPolicyHasZeroRetries()
|
||||
{
|
||||
// Tier A Write default is RetryCount=0. Even isIdempotent=true shouldn't retry
|
||||
// because the policy says not to.
|
||||
var invoker = MakeInvoker(new DriverResiliencePipelineBuilder(), new DriverResilienceOptions { Tier = DriverTier.A });
|
||||
var attempts = 0;
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await invoker.ExecuteWriteAsync(
|
||||
"host-1",
|
||||
isIdempotent: true,
|
||||
async _ =>
|
||||
{
|
||||
attempts++;
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
#pragma warning disable CS0162
|
||||
return 0;
|
||||
#pragma warning restore CS0162
|
||||
},
|
||||
CancellationToken.None));
|
||||
|
||||
attempts.ShouldBe(1, "tier-A default for Write is RetryCount=0");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Execute_HonorsDifferentHosts_Independently()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var invoker = MakeInvoker(builder, new DriverResilienceOptions { Tier = DriverTier.A });
|
||||
|
||||
await invoker.ExecuteAsync(DriverCapability.Read, "host-a", _ => ValueTask.FromResult(1), CancellationToken.None);
|
||||
await invoker.ExecuteAsync(DriverCapability.Read, "host-b", _ => ValueTask.FromResult(2), CancellationToken.None);
|
||||
|
||||
builder.CachedPipelineCount.ShouldBe(2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class DriverResilienceOptionsParserTests
|
||||
{
|
||||
[Fact]
|
||||
public void NullJson_ReturnsPureTierDefaults()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, null, out var diag);
|
||||
|
||||
diag.ShouldBeNull();
|
||||
options.Tier.ShouldBe(DriverTier.A);
|
||||
options.Resolve(DriverCapability.Read).ShouldBe(
|
||||
DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhitespaceJson_ReturnsDefaults()
|
||||
{
|
||||
DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.B, " ", out var diag);
|
||||
diag.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MalformedJson_FallsBack_WithDiagnostic()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, "{not json", out var diag);
|
||||
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("malformed");
|
||||
options.Tier.ShouldBe(DriverTier.A);
|
||||
options.Resolve(DriverCapability.Read).ShouldBe(
|
||||
DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EmptyObject_ReturnsDefaults()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, "{}", out var diag);
|
||||
|
||||
diag.ShouldBeNull();
|
||||
options.Resolve(DriverCapability.Write).ShouldBe(
|
||||
DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Write]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReadOverride_MergedIntoTierDefaults()
|
||||
{
|
||||
var json = """
|
||||
{
|
||||
"capabilityPolicies": {
|
||||
"Read": { "timeoutSeconds": 5, "retryCount": 7, "breakerFailureThreshold": 2 }
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, json, out var diag);
|
||||
|
||||
diag.ShouldBeNull();
|
||||
var read = options.Resolve(DriverCapability.Read);
|
||||
read.TimeoutSeconds.ShouldBe(5);
|
||||
read.RetryCount.ShouldBe(7);
|
||||
read.BreakerFailureThreshold.ShouldBe(2);
|
||||
|
||||
// Other capabilities untouched
|
||||
options.Resolve(DriverCapability.Write).ShouldBe(
|
||||
DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Write]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PartialPolicy_FillsMissingFieldsFromTierDefault()
|
||||
{
|
||||
var json = """
|
||||
{
|
||||
"capabilityPolicies": {
|
||||
"Read": { "retryCount": 10 }
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, json, out _);
|
||||
|
||||
var read = options.Resolve(DriverCapability.Read);
|
||||
var tierDefault = DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read];
|
||||
read.RetryCount.ShouldBe(10);
|
||||
read.TimeoutSeconds.ShouldBe(tierDefault.TimeoutSeconds, "partial override; timeout falls back to tier default");
|
||||
read.BreakerFailureThreshold.ShouldBe(tierDefault.BreakerFailureThreshold);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BulkheadOverrides_AreHonored()
|
||||
{
|
||||
var json = """
|
||||
{ "bulkheadMaxConcurrent": 100, "bulkheadMaxQueue": 500 }
|
||||
""";
|
||||
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.B, json, out _);
|
||||
|
||||
options.BulkheadMaxConcurrent.ShouldBe(100);
|
||||
options.BulkheadMaxQueue.ShouldBe(500);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnknownCapability_Surfaces_InDiagnostic_ButDoesNotFail()
|
||||
{
|
||||
var json = """
|
||||
{
|
||||
"capabilityPolicies": {
|
||||
"InventedCapability": { "timeoutSeconds": 99 }
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, json, out var diag);
|
||||
|
||||
diag.ShouldNotBeNull();
|
||||
diag.ShouldContain("InventedCapability");
|
||||
// Known capabilities untouched.
|
||||
options.Resolve(DriverCapability.Read).ShouldBe(
|
||||
DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PropertyNames_AreCaseInsensitive()
|
||||
{
|
||||
var json = """
|
||||
{ "BULKHEADMAXCONCURRENT": 42 }
|
||||
""";
|
||||
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, json, out _);
|
||||
|
||||
options.BulkheadMaxConcurrent.ShouldBe(42);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CapabilityName_IsCaseInsensitive()
|
||||
{
|
||||
var json = """
|
||||
{ "capabilityPolicies": { "read": { "retryCount": 99 } } }
|
||||
""";
|
||||
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, json, out var diag);
|
||||
|
||||
diag.ShouldBeNull();
|
||||
options.Resolve(DriverCapability.Read).RetryCount.ShouldBe(99);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DriverTier.A)]
|
||||
[InlineData(DriverTier.B)]
|
||||
[InlineData(DriverTier.C)]
|
||||
public void EveryTier_WithEmptyJson_RoundTrips_Its_Defaults(DriverTier tier)
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(tier, "{}", out var diag);
|
||||
|
||||
diag.ShouldBeNull();
|
||||
options.Tier.ShouldBe(tier);
|
||||
foreach (var cap in Enum.GetValues<DriverCapability>())
|
||||
options.Resolve(cap).ShouldBe(DriverResilienceOptions.GetTierDefaults(tier)[cap]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecycleIntervalSeconds_TierC_PositiveValue_ParsesAndSurfaces()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.C, "{\"recycleIntervalSeconds\":3600}", out var diag);
|
||||
|
||||
diag.ShouldBeNull();
|
||||
options.RecycleIntervalSeconds.ShouldBe(3600);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecycleIntervalSeconds_Null_DefaultsToNull()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.C, "{}", out _);
|
||||
options.RecycleIntervalSeconds.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DriverTier.A)]
|
||||
[InlineData(DriverTier.B)]
|
||||
public void RecycleIntervalSeconds_OnTierAorB_Rejected_With_Diagnostic(DriverTier tier)
|
||||
{
|
||||
// Decision #74 — in-process drivers must not scheduled-recycle because it would
|
||||
// tear down every OPC UA session. The parser surfaces a diagnostic rather than
|
||||
// silently honouring the value.
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
tier, "{\"recycleIntervalSeconds\":3600}", out var diag);
|
||||
|
||||
options.RecycleIntervalSeconds.ShouldBeNull();
|
||||
diag.ShouldContain("Tier C only");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecycleIntervalSeconds_NonPositive_Rejected_With_Diagnostic()
|
||||
{
|
||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||
DriverTier.C, "{\"recycleIntervalSeconds\":0}", out var diag);
|
||||
|
||||
options.RecycleIntervalSeconds.ShouldBeNull();
|
||||
diag.ShouldContain("must be positive");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class DriverResilienceOptionsTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(DriverTier.A)]
|
||||
[InlineData(DriverTier.B)]
|
||||
[InlineData(DriverTier.C)]
|
||||
public void TierDefaults_Cover_EveryCapability(DriverTier tier)
|
||||
{
|
||||
var defaults = DriverResilienceOptions.GetTierDefaults(tier);
|
||||
|
||||
foreach (var capability in Enum.GetValues<DriverCapability>())
|
||||
defaults.ShouldContainKey(capability);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DriverTier.A)]
|
||||
[InlineData(DriverTier.B)]
|
||||
[InlineData(DriverTier.C)]
|
||||
public void Write_NeverRetries_ByDefault(DriverTier tier)
|
||||
{
|
||||
var defaults = DriverResilienceOptions.GetTierDefaults(tier);
|
||||
defaults[DriverCapability.Write].RetryCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DriverTier.A)]
|
||||
[InlineData(DriverTier.B)]
|
||||
[InlineData(DriverTier.C)]
|
||||
public void AlarmAcknowledge_NeverRetries_ByDefault(DriverTier tier)
|
||||
{
|
||||
var defaults = DriverResilienceOptions.GetTierDefaults(tier);
|
||||
defaults[DriverCapability.AlarmAcknowledge].RetryCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DriverTier.A, DriverCapability.Read)]
|
||||
[InlineData(DriverTier.A, DriverCapability.HistoryRead)]
|
||||
[InlineData(DriverTier.B, DriverCapability.Discover)]
|
||||
[InlineData(DriverTier.B, DriverCapability.Probe)]
|
||||
[InlineData(DriverTier.C, DriverCapability.AlarmSubscribe)]
|
||||
public void IdempotentCapabilities_Retry_ByDefault(DriverTier tier, DriverCapability capability)
|
||||
{
|
||||
var defaults = DriverResilienceOptions.GetTierDefaults(tier);
|
||||
defaults[capability].RetryCount.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TierC_DisablesCircuitBreaker_DeferringToSupervisor()
|
||||
{
|
||||
var defaults = DriverResilienceOptions.GetTierDefaults(DriverTier.C);
|
||||
|
||||
foreach (var (_, policy) in defaults)
|
||||
policy.BreakerFailureThreshold.ShouldBe(0, "Tier C breaker is handled by the Proxy supervisor (decision #68)");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DriverTier.A)]
|
||||
[InlineData(DriverTier.B)]
|
||||
public void TierAAndB_EnableCircuitBreaker(DriverTier tier)
|
||||
{
|
||||
var defaults = DriverResilienceOptions.GetTierDefaults(tier);
|
||||
|
||||
foreach (var (_, policy) in defaults)
|
||||
policy.BreakerFailureThreshold.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_Uses_TierDefaults_When_NoOverride()
|
||||
{
|
||||
var options = new DriverResilienceOptions { Tier = DriverTier.A };
|
||||
|
||||
var resolved = options.Resolve(DriverCapability.Read);
|
||||
|
||||
resolved.ShouldBe(DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_Uses_Override_When_Configured()
|
||||
{
|
||||
var custom = new CapabilityPolicy(TimeoutSeconds: 42, RetryCount: 7, BreakerFailureThreshold: 9);
|
||||
var options = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Read] = custom,
|
||||
},
|
||||
};
|
||||
|
||||
options.Resolve(DriverCapability.Read).ShouldBe(custom);
|
||||
options.Resolve(DriverCapability.Write).ShouldBe(
|
||||
DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Write]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
using Polly.CircuitBreaker;
|
||||
using Polly.Timeout;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class DriverResiliencePipelineBuilderTests
|
||||
{
|
||||
private static readonly DriverResilienceOptions TierAOptions = new() { Tier = DriverTier.A };
|
||||
|
||||
[Fact]
|
||||
public async Task Read_Retries_Transient_Failures()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions);
|
||||
var attempts = 0;
|
||||
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
attempts++;
|
||||
if (attempts < 3) throw new InvalidOperationException("transient");
|
||||
await Task.Yield();
|
||||
});
|
||||
|
||||
attempts.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Write_DoesNotRetry_OnFailure()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions);
|
||||
var attempts = 0;
|
||||
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
{
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
attempts++;
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
});
|
||||
});
|
||||
|
||||
attempts.ShouldBe(1);
|
||||
ex.Message.ShouldBe("boom");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AlarmAcknowledge_DoesNotRetry_OnFailure()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.AlarmAcknowledge, TierAOptions);
|
||||
var attempts = 0;
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
{
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
attempts++;
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
});
|
||||
});
|
||||
|
||||
attempts.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pipeline_IsIsolated_PerHost()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var driverId = "drv-test";
|
||||
|
||||
var hostA = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
||||
var hostB = builder.GetOrCreate(driverId, "host-b", DriverCapability.Read, TierAOptions);
|
||||
|
||||
hostA.ShouldNotBeSameAs(hostB);
|
||||
builder.CachedPipelineCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pipeline_IsReused_ForSameTriple()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var driverId = "drv-test";
|
||||
|
||||
var first = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
||||
var second = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
||||
|
||||
first.ShouldBeSameAs(second);
|
||||
builder.CachedPipelineCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pipeline_IsIsolated_PerCapability()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var driverId = "drv-test";
|
||||
|
||||
var read = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
|
||||
var write = builder.GetOrCreate(driverId, "host-a", DriverCapability.Write, TierAOptions);
|
||||
|
||||
read.ShouldNotBeSameAs(write);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeadHost_DoesNotOpenBreaker_ForSiblingHost()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var driverId = "drv-test";
|
||||
|
||||
var deadHost = builder.GetOrCreate(driverId, "dead-plc", DriverCapability.Read, TierAOptions);
|
||||
var liveHost = builder.GetOrCreate(driverId, "live-plc", DriverCapability.Read, TierAOptions);
|
||||
|
||||
var threshold = TierAOptions.Resolve(DriverCapability.Read).BreakerFailureThreshold;
|
||||
for (var i = 0; i < threshold + 5; i++)
|
||||
{
|
||||
await Should.ThrowAsync<Exception>(async () =>
|
||||
await deadHost.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("dead plc");
|
||||
}));
|
||||
}
|
||||
|
||||
var liveAttempts = 0;
|
||||
await liveHost.ExecuteAsync(async _ =>
|
||||
{
|
||||
liveAttempts++;
|
||||
await Task.Yield();
|
||||
});
|
||||
|
||||
liveAttempts.ShouldBe(1, "healthy sibling host must not be affected by dead peer");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CircuitBreaker_Opens_AfterFailureThreshold_OnTierA()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions);
|
||||
|
||||
var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold;
|
||||
for (var i = 0; i < threshold; i++)
|
||||
{
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
}));
|
||||
}
|
||||
|
||||
await Should.ThrowAsync<BrokenCircuitException>(async () =>
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
}));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Timeout_Cancels_SlowOperation()
|
||||
{
|
||||
var tierAWithShortTimeout = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Read] = new(TimeoutSeconds: 1, RetryCount: 0, BreakerFailureThreshold: 5),
|
||||
},
|
||||
};
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, tierAWithShortTimeout);
|
||||
|
||||
await Should.ThrowAsync<TimeoutRejectedException>(async () =>
|
||||
await pipeline.ExecuteAsync(async ct =>
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), ct);
|
||||
}));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Invalidate_Removes_OnlyMatchingInstance()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var keepId = "drv-keep";
|
||||
var dropId = "drv-drop";
|
||||
|
||||
builder.GetOrCreate(keepId, "h", DriverCapability.Read, TierAOptions);
|
||||
builder.GetOrCreate(keepId, "h", DriverCapability.Write, TierAOptions);
|
||||
builder.GetOrCreate(dropId, "h", DriverCapability.Read, TierAOptions);
|
||||
|
||||
var removed = builder.Invalidate(dropId);
|
||||
|
||||
removed.ShouldBe(1);
|
||||
builder.CachedPipelineCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cancellation_IsNot_Retried()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions);
|
||||
var attempts = 0;
|
||||
using var cts = new CancellationTokenSource();
|
||||
cts.Cancel();
|
||||
|
||||
await Should.ThrowAsync<OperationCanceledException>(async () =>
|
||||
await pipeline.ExecuteAsync(async ct =>
|
||||
{
|
||||
attempts++;
|
||||
ct.ThrowIfCancellationRequested();
|
||||
await Task.Yield();
|
||||
}, cts.Token));
|
||||
|
||||
attempts.ShouldBeLessThanOrEqualTo(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tracker_RecordsFailure_OnEveryRetry()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
||||
var pipeline = builder.GetOrCreate("drv-trk", "host-x", DriverCapability.Read, TierAOptions);
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("always fails");
|
||||
}));
|
||||
|
||||
var snap = tracker.TryGet("drv-trk", "host-x");
|
||||
snap.ShouldNotBeNull();
|
||||
var retryCount = TierAOptions.Resolve(DriverCapability.Read).RetryCount;
|
||||
snap!.ConsecutiveFailures.ShouldBe(retryCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tracker_StampsBreakerOpen_WhenBreakerTrips()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
||||
var pipeline = builder.GetOrCreate("drv-trk", "host-b", DriverCapability.Write, TierAOptions);
|
||||
|
||||
var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold;
|
||||
for (var i = 0; i < threshold; i++)
|
||||
{
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await pipeline.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("boom");
|
||||
}));
|
||||
}
|
||||
|
||||
var snap = tracker.TryGet("drv-trk", "host-b");
|
||||
snap.ShouldNotBeNull();
|
||||
snap!.LastBreakerOpenUtc.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tracker_IsolatesCounters_PerHost()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
||||
var dead = builder.GetOrCreate("drv-trk", "dead", DriverCapability.Read, TierAOptions);
|
||||
var live = builder.GetOrCreate("drv-trk", "live", DriverCapability.Read, TierAOptions);
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await dead.ExecuteAsync(async _ =>
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("dead");
|
||||
}));
|
||||
await live.ExecuteAsync(async _ => await Task.Yield());
|
||||
|
||||
tracker.TryGet("drv-trk", "dead")!.ConsecutiveFailures.ShouldBeGreaterThan(0);
|
||||
tracker.TryGet("drv-trk", "live").ShouldBeNull();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class DriverResilienceStatusTrackerTests
|
||||
{
|
||||
private static readonly DateTime Now = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public void TryGet_Returns_Null_Before_AnyWrite()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
|
||||
tracker.TryGet("drv", "host").ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_Accumulates_ConsecutiveFailures()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
|
||||
tracker.RecordFailure("drv", "host", Now);
|
||||
tracker.RecordFailure("drv", "host", Now.AddSeconds(1));
|
||||
tracker.RecordFailure("drv", "host", Now.AddSeconds(2));
|
||||
|
||||
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordSuccess_Resets_ConsecutiveFailures()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
tracker.RecordFailure("drv", "host", Now);
|
||||
tracker.RecordFailure("drv", "host", Now.AddSeconds(1));
|
||||
|
||||
tracker.RecordSuccess("drv", "host", Now.AddSeconds(2));
|
||||
|
||||
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordBreakerOpen_Populates_LastBreakerOpenUtc()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
|
||||
tracker.RecordBreakerOpen("drv", "host", Now);
|
||||
|
||||
tracker.TryGet("drv", "host")!.LastBreakerOpenUtc.ShouldBe(Now);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRecycle_Populates_LastRecycleUtc()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
|
||||
tracker.RecordRecycle("drv", "host", Now);
|
||||
|
||||
tracker.TryGet("drv", "host")!.LastRecycleUtc.ShouldBe(Now);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFootprint_CapturesBaselineAndCurrent()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
|
||||
tracker.RecordFootprint("drv", "host", baselineBytes: 100_000_000, currentBytes: 150_000_000, Now);
|
||||
|
||||
var snap = tracker.TryGet("drv", "host")!;
|
||||
snap.BaselineFootprintBytes.ShouldBe(100_000_000);
|
||||
snap.CurrentFootprintBytes.ShouldBe(150_000_000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DifferentHosts_AreIndependent()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
|
||||
tracker.RecordFailure("drv", "host-a", Now);
|
||||
tracker.RecordFailure("drv", "host-b", Now);
|
||||
tracker.RecordSuccess("drv", "host-a", Now.AddSeconds(1));
|
||||
|
||||
tracker.TryGet("drv", "host-a")!.ConsecutiveFailures.ShouldBe(0);
|
||||
tracker.TryGet("drv", "host-b")!.ConsecutiveFailures.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Snapshot_ReturnsAll_TrackedPairs()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
tracker.RecordFailure("drv-1", "host-a", Now);
|
||||
tracker.RecordFailure("drv-1", "host-b", Now);
|
||||
tracker.RecordFailure("drv-2", "host-a", Now);
|
||||
|
||||
var snapshot = tracker.Snapshot();
|
||||
|
||||
snapshot.Count.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConcurrentWrites_DoNotLose_Failures()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
Parallel.For(0, 500, _ => tracker.RecordFailure("drv", "host", Now));
|
||||
|
||||
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(500);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for the Phase 6.1 Stream A.5 contract — wrapping a flaky
|
||||
/// <see cref="IReadable"/> / <see cref="IWritable"/> through the <see cref="CapabilityInvoker"/>.
|
||||
/// Exercises the three scenarios the plan enumerates: transient read succeeds after N
|
||||
/// retries; non-idempotent write fails after one attempt; idempotent write retries through.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class FlakeyDriverIntegrationTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Read_SurfacesSuccess_AfterTransientFailures()
|
||||
{
|
||||
var flaky = new FlakeyDriver(failReadsBeforeIndex: 5);
|
||||
var options = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
// TimeoutSeconds=30 gives slack for 5 exponential-backoff retries under
|
||||
// parallel-test-execution CPU pressure; 10 retries at the default Delay=100ms
|
||||
// exponential can otherwise exceed a 2-second budget intermittently.
|
||||
[DriverCapability.Read] = new(TimeoutSeconds: 30, RetryCount: 10, BreakerFailureThreshold: 50),
|
||||
},
|
||||
};
|
||||
var invoker = new CapabilityInvoker(new DriverResiliencePipelineBuilder(), "drv-test", () => options);
|
||||
|
||||
var result = await invoker.ExecuteAsync(
|
||||
DriverCapability.Read,
|
||||
"host-1",
|
||||
async ct => await flaky.ReadAsync(["tag-a"], ct),
|
||||
CancellationToken.None);
|
||||
|
||||
flaky.ReadAttempts.ShouldBe(6);
|
||||
result[0].StatusCode.ShouldBe(0u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Write_NonIdempotent_FailsOnFirstFailure_NoReplay()
|
||||
{
|
||||
var flaky = new FlakeyDriver(failWritesBeforeIndex: 3);
|
||||
var optionsWithAggressiveRetry = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Write] = new(TimeoutSeconds: 2, RetryCount: 5, BreakerFailureThreshold: 50),
|
||||
},
|
||||
};
|
||||
var invoker = new CapabilityInvoker(new DriverResiliencePipelineBuilder(), "drv-test", () => optionsWithAggressiveRetry);
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await invoker.ExecuteWriteAsync(
|
||||
"host-1",
|
||||
isIdempotent: false,
|
||||
async ct => await flaky.WriteAsync([new WriteRequest("pulse-coil", true)], ct),
|
||||
CancellationToken.None));
|
||||
|
||||
flaky.WriteAttempts.ShouldBe(1, "non-idempotent write must never replay (decision #44)");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Write_Idempotent_RetriesUntilSuccess()
|
||||
{
|
||||
var flaky = new FlakeyDriver(failWritesBeforeIndex: 2);
|
||||
var optionsWithRetry = new DriverResilienceOptions
|
||||
{
|
||||
Tier = DriverTier.A,
|
||||
CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
|
||||
{
|
||||
[DriverCapability.Write] = new(TimeoutSeconds: 2, RetryCount: 5, BreakerFailureThreshold: 50),
|
||||
},
|
||||
};
|
||||
var invoker = new CapabilityInvoker(new DriverResiliencePipelineBuilder(), "drv-test", () => optionsWithRetry);
|
||||
|
||||
var results = await invoker.ExecuteWriteAsync(
|
||||
"host-1",
|
||||
isIdempotent: true,
|
||||
async ct => await flaky.WriteAsync([new WriteRequest("set-point", 42.0f)], ct),
|
||||
CancellationToken.None);
|
||||
|
||||
flaky.WriteAttempts.ShouldBe(3);
|
||||
results[0].StatusCode.ShouldBe(0u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MultipleHosts_OnOneDriver_HaveIndependentFailureCounts()
|
||||
{
|
||||
var flaky = new FlakeyDriver(failReadsBeforeIndex: 0);
|
||||
var options = new DriverResilienceOptions { Tier = DriverTier.A };
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var invoker = new CapabilityInvoker(builder, "drv-test", () => options);
|
||||
|
||||
// host-dead: force many failures to exhaust retries + trip breaker
|
||||
var threshold = options.Resolve(DriverCapability.Read).BreakerFailureThreshold;
|
||||
for (var i = 0; i < threshold + 5; i++)
|
||||
{
|
||||
await Should.ThrowAsync<Exception>(async () =>
|
||||
await invoker.ExecuteAsync(DriverCapability.Read, "host-dead",
|
||||
_ => throw new InvalidOperationException("dead"),
|
||||
CancellationToken.None));
|
||||
}
|
||||
|
||||
// host-live: succeeds on first call — unaffected by the dead-host breaker
|
||||
var liveAttempts = 0;
|
||||
await invoker.ExecuteAsync(DriverCapability.Read, "host-live",
|
||||
_ => { liveAttempts++; return ValueTask.FromResult("ok"); },
|
||||
CancellationToken.None);
|
||||
|
||||
liveAttempts.ShouldBe(1);
|
||||
}
|
||||
|
||||
private sealed class FlakeyDriver : IReadable, IWritable
|
||||
{
|
||||
private readonly int _failReadsBeforeIndex;
|
||||
private readonly int _failWritesBeforeIndex;
|
||||
|
||||
public int ReadAttempts { get; private set; }
|
||||
public int WriteAttempts { get; private set; }
|
||||
|
||||
public FlakeyDriver(int failReadsBeforeIndex = 0, int failWritesBeforeIndex = 0)
|
||||
{
|
||||
_failReadsBeforeIndex = failReadsBeforeIndex;
|
||||
_failWritesBeforeIndex = failWritesBeforeIndex;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(
|
||||
IReadOnlyList<string> fullReferences,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var attempt = ++ReadAttempts;
|
||||
if (attempt <= _failReadsBeforeIndex)
|
||||
throw new InvalidOperationException($"transient read failure #{attempt}");
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
IReadOnlyList<DataValueSnapshot> result = fullReferences
|
||||
.Select(_ => new DataValueSnapshot(Value: 0, StatusCode: 0u, SourceTimestampUtc: now, ServerTimestampUtc: now))
|
||||
.ToList();
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<WriteResult>> WriteAsync(
|
||||
IReadOnlyList<WriteRequest> writes,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var attempt = ++WriteAttempts;
|
||||
if (attempt <= _failWritesBeforeIndex)
|
||||
throw new InvalidOperationException($"transient write failure #{attempt}");
|
||||
|
||||
IReadOnlyList<WriteResult> result = writes.Select(_ => new WriteResult(0u)).ToList();
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class InFlightCounterTests
|
||||
{
|
||||
[Fact]
|
||||
public void StartThenComplete_NetsToZero()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
tracker.RecordCallStart("drv", "host-a");
|
||||
tracker.RecordCallComplete("drv", "host-a");
|
||||
|
||||
tracker.TryGet("drv", "host-a")!.CurrentInFlight.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NestedStarts_SumDepth()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
tracker.RecordCallStart("drv", "host-a");
|
||||
tracker.RecordCallStart("drv", "host-a");
|
||||
tracker.RecordCallStart("drv", "host-a");
|
||||
|
||||
tracker.TryGet("drv", "host-a")!.CurrentInFlight.ShouldBe(3);
|
||||
|
||||
tracker.RecordCallComplete("drv", "host-a");
|
||||
tracker.TryGet("drv", "host-a")!.CurrentInFlight.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CompleteBeforeStart_ClampedToZero()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
tracker.RecordCallComplete("drv", "host-a");
|
||||
|
||||
// A stray Complete without a matching Start shouldn't drive the counter negative.
|
||||
tracker.TryGet("drv", "host-a")!.CurrentInFlight.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DifferentHosts_TrackIndependently()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
tracker.RecordCallStart("drv", "host-a");
|
||||
tracker.RecordCallStart("drv", "host-a");
|
||||
tracker.RecordCallStart("drv", "host-b");
|
||||
|
||||
tracker.TryGet("drv", "host-a")!.CurrentInFlight.ShouldBe(2);
|
||||
tracker.TryGet("drv", "host-b")!.CurrentInFlight.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConcurrentStarts_DoNotLose_Count()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
Parallel.For(0, 500, _ => tracker.RecordCallStart("drv", "host-a"));
|
||||
|
||||
tracker.TryGet("drv", "host-a")!.CurrentInFlight.ShouldBe(500);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CapabilityInvoker_IncrementsTracker_DuringExecution()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
var invoker = new CapabilityInvoker(
|
||||
new DriverResiliencePipelineBuilder(),
|
||||
"drv-live",
|
||||
() => new DriverResilienceOptions { Tier = DriverTier.A },
|
||||
driverType: "Modbus",
|
||||
statusTracker: tracker);
|
||||
|
||||
var observedMidCall = -1;
|
||||
await invoker.ExecuteAsync(
|
||||
DriverCapability.Read,
|
||||
"plc-1",
|
||||
async _ =>
|
||||
{
|
||||
observedMidCall = tracker.TryGet("drv-live", "plc-1")?.CurrentInFlight ?? -1;
|
||||
await Task.Yield();
|
||||
return 42;
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
observedMidCall.ShouldBe(1, "during call, in-flight == 1");
|
||||
tracker.TryGet("drv-live", "plc-1")!.CurrentInFlight.ShouldBe(0, "post-call, counter decremented");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CapabilityInvoker_ExceptionPath_DecrementsCounter()
|
||||
{
|
||||
var tracker = new DriverResilienceStatusTracker();
|
||||
var invoker = new CapabilityInvoker(
|
||||
new DriverResiliencePipelineBuilder(),
|
||||
"drv-live",
|
||||
() => new DriverResilienceOptions { Tier = DriverTier.A },
|
||||
statusTracker: tracker);
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||
await invoker.ExecuteAsync<int>(
|
||||
DriverCapability.Write,
|
||||
"plc-1",
|
||||
_ => throw new InvalidOperationException("boom"),
|
||||
CancellationToken.None));
|
||||
|
||||
tracker.TryGet("drv-live", "plc-1")!.CurrentInFlight.ShouldBe(0,
|
||||
"finally-block must decrement even when call-site throws");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CapabilityInvoker_WithoutTracker_DoesNotThrow()
|
||||
{
|
||||
var invoker = new CapabilityInvoker(
|
||||
new DriverResiliencePipelineBuilder(),
|
||||
"drv-live",
|
||||
() => new DriverResilienceOptions { Tier = DriverTier.A },
|
||||
statusTracker: null);
|
||||
|
||||
var result = await invoker.ExecuteAsync(
|
||||
DriverCapability.Read, "host-1",
|
||||
_ => ValueTask.FromResult(7),
|
||||
CancellationToken.None);
|
||||
|
||||
result.ShouldBe(7);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||
|
||||
/// <summary>
|
||||
/// Exercises the per-call host resolver contract against the shared
|
||||
/// <see cref="DriverResiliencePipelineBuilder"/> + <see cref="CapabilityInvoker"/> — one
|
||||
/// dead PLC behind a multi-device driver must NOT open the breaker for healthy sibling
|
||||
/// PLCs (decision #144).
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class PerCallHostResolverDispatchTests
|
||||
{
|
||||
private sealed class StaticResolver : IPerCallHostResolver
|
||||
{
|
||||
private readonly Dictionary<string, string> _map;
|
||||
public StaticResolver(Dictionary<string, string> map) => _map = map;
|
||||
public string ResolveHost(string fullReference) =>
|
||||
_map.TryGetValue(fullReference, out var host) ? host : string.Empty;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeadPlc_DoesNotOpenBreaker_For_HealthyPlc_With_Resolver()
|
||||
{
|
||||
// Two PLCs behind one driver. Dead PLC keeps failing; healthy PLC must keep serving.
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var options = new DriverResilienceOptions { Tier = DriverTier.B };
|
||||
var invoker = new CapabilityInvoker(builder, "drv-modbus", () => options);
|
||||
|
||||
var resolver = new StaticResolver(new Dictionary<string, string>
|
||||
{
|
||||
["tag-on-dead"] = "plc-dead",
|
||||
["tag-on-alive"] = "plc-alive",
|
||||
});
|
||||
|
||||
var threshold = options.Resolve(DriverCapability.Read).BreakerFailureThreshold;
|
||||
for (var i = 0; i < threshold + 3; i++)
|
||||
{
|
||||
await Should.ThrowAsync<Exception>(async () =>
|
||||
await invoker.ExecuteAsync(
|
||||
DriverCapability.Read,
|
||||
hostName: resolver.ResolveHost("tag-on-dead"),
|
||||
_ => throw new InvalidOperationException("plc-dead unreachable"),
|
||||
CancellationToken.None));
|
||||
}
|
||||
|
||||
// Healthy PLC's pipeline is in a different bucket; the first call should succeed
|
||||
// without hitting the dead-PLC breaker.
|
||||
var aliveAttempts = 0;
|
||||
await invoker.ExecuteAsync(
|
||||
DriverCapability.Read,
|
||||
hostName: resolver.ResolveHost("tag-on-alive"),
|
||||
_ => { aliveAttempts++; return ValueTask.FromResult("ok"); },
|
||||
CancellationToken.None);
|
||||
|
||||
aliveAttempts.ShouldBe(1, "decision #144 — per-PLC isolation keeps healthy PLCs serving");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolver_EmptyString_Treated_As_Single_Host_Fallback()
|
||||
{
|
||||
var resolver = new StaticResolver(new Dictionary<string, string>
|
||||
{
|
||||
["tag-unknown"] = "",
|
||||
});
|
||||
|
||||
resolver.ResolveHost("tag-unknown").ShouldBe("");
|
||||
resolver.ResolveHost("not-in-map").ShouldBe("", "unknown refs return empty so dispatch falls back to single-host");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WithoutResolver_SameHost_Shares_One_Pipeline()
|
||||
{
|
||||
// Without a resolver all calls share the DriverInstanceId pipeline — that's the
|
||||
// pre-decision-#144 behavior single-host drivers should keep.
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var options = new DriverResilienceOptions { Tier = DriverTier.A };
|
||||
var invoker = new CapabilityInvoker(builder, "drv-single", () => options);
|
||||
|
||||
await invoker.ExecuteAsync(DriverCapability.Read, "drv-single",
|
||||
_ => ValueTask.FromResult("a"), CancellationToken.None);
|
||||
await invoker.ExecuteAsync(DriverCapability.Read, "drv-single",
|
||||
_ => ValueTask.FromResult("b"), CancellationToken.None);
|
||||
|
||||
builder.CachedPipelineCount.ShouldBe(1, "single-host drivers share one pipeline");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WithResolver_TwoHosts_Get_Two_Pipelines()
|
||||
{
|
||||
var builder = new DriverResiliencePipelineBuilder();
|
||||
var options = new DriverResilienceOptions { Tier = DriverTier.B };
|
||||
var invoker = new CapabilityInvoker(builder, "drv-modbus", () => options);
|
||||
var resolver = new StaticResolver(new Dictionary<string, string>
|
||||
{
|
||||
["tag-a"] = "plc-a",
|
||||
["tag-b"] = "plc-b",
|
||||
});
|
||||
|
||||
await invoker.ExecuteAsync(DriverCapability.Read, resolver.ResolveHost("tag-a"),
|
||||
_ => ValueTask.FromResult(1), CancellationToken.None);
|
||||
await invoker.ExecuteAsync(DriverCapability.Read, resolver.ResolveHost("tag-b"),
|
||||
_ => ValueTask.FromResult(2), CancellationToken.None);
|
||||
|
||||
builder.CachedPipelineCount.ShouldBe(2, "each host keyed on its own pipeline");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user