Merge #875 Galaxy initialize 20 s budget (per-driver-type capability defaults)
v2-ci / unit-tests (push) Successful in 1h14m54s
v2-ci / unit-tests (push) Successful in 1h14m54s
Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -134,7 +134,12 @@ following this note. `DriverResilienceOptions.GetTierDefaults(tier)` supplies
|
|||||||
the real per-capability timeout / retry / breaker policies, resolved via
|
the real per-capability timeout / retry / breaker policies, resolved via
|
||||||
`DriverFactoryRegistry.GetTier` and applied by `DriverCapabilityInvokerFactory`.
|
`DriverFactoryRegistry.GetTier` and applied by `DriverCapabilityInvokerFactory`.
|
||||||
Every driver runs Tier A because no factory passes a tier, so today that means
|
Every driver runs Tier A because no factory passes a tier, so today that means
|
||||||
one uniform policy set.
|
one uniform policy set — except where a driver type registers **capability
|
||||||
|
defaults** with its factory (`DriverFactoryRegistry.Register(...,
|
||||||
|
capabilityDefaults:)`, read back by `GetCapabilityDefaults`). Those layer over
|
||||||
|
the tier table and under any per-instance `ResilienceConfig`. Galaxy registers
|
||||||
|
`Initialize` = 20 s (#875): its session open plus platform seed took 4–5 s
|
||||||
|
against a real gateway, so Tier A's 5 s failed a quarter of restarts.
|
||||||
|
|
||||||
`IDriver` **no longer carries `GetMemoryFootprint()` or
|
`IDriver` **no longer carries `GetMemoryFootprint()` or
|
||||||
`FlushOptionalCachesAsync()`** (Gitea **#525**). They existed to feed
|
`FlushOptionalCachesAsync()`** (Gitea **#525**). They existed to feed
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.OtOpcUa.Core.Hosting;
|
namespace ZB.MOM.WW.OtOpcUa.Core.Hosting;
|
||||||
@@ -21,6 +22,8 @@ public sealed class DriverFactoryRegistry
|
|||||||
= new(StringComparer.OrdinalIgnoreCase);
|
= new(StringComparer.OrdinalIgnoreCase);
|
||||||
private readonly Dictionary<string, DriverTier> _tiers
|
private readonly Dictionary<string, DriverTier> _tiers
|
||||||
= new(StringComparer.OrdinalIgnoreCase);
|
= new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
private readonly Dictionary<string, IReadOnlyDictionary<DriverCapability, CapabilityPolicy>> _capabilityDefaults
|
||||||
|
= new(StringComparer.OrdinalIgnoreCase);
|
||||||
private readonly object _lock = new();
|
private readonly object _lock = new();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -42,8 +45,14 @@ public sealed class DriverFactoryRegistry
|
|||||||
/// <c>DriverInstanceBootstrapper</c> can skip in-process drivers by tier check
|
/// <c>DriverInstanceBootstrapper</c> can skip in-process drivers by tier check
|
||||||
/// rather than by driver-type allow-list.
|
/// rather than by driver-type allow-list.
|
||||||
/// </param>
|
/// </param>
|
||||||
|
/// <param name="capabilityDefaults">
|
||||||
|
/// Optional per-driver-TYPE capability policies layered over the tier defaults (and under any
|
||||||
|
/// per-instance <c>ResilienceConfig</c>) — for a driver whose real cost of one capability does not fit its
|
||||||
|
/// tier's table, e.g. Galaxy's initialize (session open + platform seed) against a real gateway (#875).
|
||||||
|
/// </param>
|
||||||
public void Register(string driverType, Func<string, string, IDriver> factory,
|
public void Register(string driverType, Func<string, string, IDriver> factory,
|
||||||
DriverTier tier = DriverTier.A)
|
DriverTier tier = DriverTier.A,
|
||||||
|
IReadOnlyDictionary<DriverCapability, CapabilityPolicy>? capabilityDefaults = null)
|
||||||
{
|
{
|
||||||
ArgumentException.ThrowIfNullOrWhiteSpace(driverType);
|
ArgumentException.ThrowIfNullOrWhiteSpace(driverType);
|
||||||
ArgumentNullException.ThrowIfNull(factory);
|
ArgumentNullException.ThrowIfNull(factory);
|
||||||
@@ -54,6 +63,7 @@ public sealed class DriverFactoryRegistry
|
|||||||
$"DriverType '{driverType}' factory already registered for this process");
|
$"DriverType '{driverType}' factory already registered for this process");
|
||||||
_factories[driverType] = factory;
|
_factories[driverType] = factory;
|
||||||
_tiers[driverType] = tier;
|
_tiers[driverType] = tier;
|
||||||
|
if (capabilityDefaults is { Count: > 0 }) _capabilityDefaults[driverType] = capabilityDefaults;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,6 +93,15 @@ public sealed class DriverFactoryRegistry
|
|||||||
lock (_lock) return _tiers.GetValueOrDefault(driverType, DriverTier.A);
|
lock (_lock) return _tiers.GetValueOrDefault(driverType, DriverTier.A);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>The per-driver-type capability policy defaults registered with the factory, or null (#875).</summary>
|
||||||
|
/// <param name="driverType">The driver type to look up.</param>
|
||||||
|
/// <returns>The registered defaults, or <see langword="null"/> when the type registered none.</returns>
|
||||||
|
public IReadOnlyDictionary<DriverCapability, CapabilityPolicy>? GetCapabilityDefaults(string driverType)
|
||||||
|
{
|
||||||
|
ArgumentException.ThrowIfNullOrWhiteSpace(driverType);
|
||||||
|
lock (_lock) return _capabilityDefaults.GetValueOrDefault(driverType);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Gets the collection of registered driver type names.</summary>
|
/// <summary>Gets the collection of registered driver type names.</summary>
|
||||||
public IReadOnlyCollection<string> RegisteredTypes
|
public IReadOnlyCollection<string> RegisteredTypes
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
|||||||
private readonly Func<string, DriverTier> _tierResolver;
|
private readonly Func<string, DriverTier> _tierResolver;
|
||||||
private readonly ILogger? _logger;
|
private readonly ILogger? _logger;
|
||||||
private readonly TimeSpan? _writeTimeoutCeiling;
|
private readonly TimeSpan? _writeTimeoutCeiling;
|
||||||
|
private readonly Func<string, IReadOnlyDictionary<DriverCapability, CapabilityPolicy>?>? _typeDefaultsResolver;
|
||||||
private long _generation;
|
private long _generation;
|
||||||
|
|
||||||
/// <summary>Construct the factory over the shared resilience infrastructure.</summary>
|
/// <summary>Construct the factory over the shared resilience infrastructure.</summary>
|
||||||
@@ -46,12 +47,15 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
|||||||
/// cancels every write before the Write policy's own timeout can (lmxopcua#777). When non-null, a driver whose
|
/// cancels every write before the Write policy's own timeout can (lmxopcua#777). When non-null, a driver whose
|
||||||
/// effective Write timeout exceeds it is logged as a warning at <see cref="Create"/> time: that policy timeout
|
/// effective Write timeout exceeds it is logged as a warning at <see cref="Create"/> time: that policy timeout
|
||||||
/// never takes effect. Null skips the check.</param>
|
/// never takes effect. Null skips the check.</param>
|
||||||
|
/// <param name="typeDefaultsResolver">Resolves a driver type's registered capability defaults
|
||||||
|
/// (<c>DriverFactoryRegistry.GetCapabilityDefaults</c>, #875), layered over the tier table. Null = tier only.</param>
|
||||||
public DriverCapabilityInvokerFactory(
|
public DriverCapabilityInvokerFactory(
|
||||||
DriverResiliencePipelineBuilder builder,
|
DriverResiliencePipelineBuilder builder,
|
||||||
DriverResilienceStatusTracker statusTracker,
|
DriverResilienceStatusTracker statusTracker,
|
||||||
Func<string, DriverTier> tierResolver,
|
Func<string, DriverTier> tierResolver,
|
||||||
ILogger? logger = null,
|
ILogger? logger = null,
|
||||||
TimeSpan? writeTimeoutCeiling = null)
|
TimeSpan? writeTimeoutCeiling = null,
|
||||||
|
Func<string, IReadOnlyDictionary<DriverCapability, CapabilityPolicy>?>? typeDefaultsResolver = null)
|
||||||
{
|
{
|
||||||
ArgumentNullException.ThrowIfNull(builder);
|
ArgumentNullException.ThrowIfNull(builder);
|
||||||
ArgumentNullException.ThrowIfNull(statusTracker);
|
ArgumentNullException.ThrowIfNull(statusTracker);
|
||||||
@@ -61,13 +65,15 @@ public sealed class DriverCapabilityInvokerFactory : IDriverCapabilityInvokerFac
|
|||||||
_tierResolver = tierResolver;
|
_tierResolver = tierResolver;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
_writeTimeoutCeiling = writeTimeoutCeiling;
|
_writeTimeoutCeiling = writeTimeoutCeiling;
|
||||||
|
_typeDefaultsResolver = typeDefaultsResolver;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public IDriverCapabilityInvoker Create(string driverInstanceId, string driverType, string? resilienceConfigJson = null)
|
public IDriverCapabilityInvoker Create(string driverInstanceId, string driverType, string? resilienceConfigJson = null)
|
||||||
{
|
{
|
||||||
var tier = _tierResolver(driverType);
|
var tier = _tierResolver(driverType);
|
||||||
var options = DriverResilienceOptionsParser.ParseOrDefaults(tier, resilienceConfigJson, out var diagnostic);
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
tier, resilienceConfigJson, _typeDefaultsResolver?.Invoke(driverType), out var diagnostic);
|
||||||
if (diagnostic is not null)
|
if (diagnostic is not null)
|
||||||
{
|
{
|
||||||
_logger?.LogWarning(
|
_logger?.LogWarning(
|
||||||
|
|||||||
@@ -52,10 +52,32 @@ public static class DriverResilienceOptionsParser
|
|||||||
public static DriverResilienceOptions ParseOrDefaults(
|
public static DriverResilienceOptions ParseOrDefaults(
|
||||||
DriverTier tier,
|
DriverTier tier,
|
||||||
string? resilienceConfigJson,
|
string? resilienceConfigJson,
|
||||||
|
out string? parseDiagnostic) => ParseOrDefaults(tier, resilienceConfigJson, typeDefaults: null, out parseDiagnostic);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// As <see cref="ParseOrDefaults(DriverTier, string?, out string?)"/>, with the driver TYPE's registered
|
||||||
|
/// capability defaults (<c>DriverFactoryRegistry.GetCapabilityDefaults</c>, #875) layered over the tier
|
||||||
|
/// table before the per-instance JSON is applied.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="tier">The driver's stability tier.</param>
|
||||||
|
/// <param name="resilienceConfigJson">The per-instance ResilienceConfig JSON, or null.</param>
|
||||||
|
/// <param name="typeDefaults">The driver type's capability defaults, or null.</param>
|
||||||
|
/// <param name="parseDiagnostic">A diagnostic when the JSON was malformed or partly ignored.</param>
|
||||||
|
/// <returns>The effective options.</returns>
|
||||||
|
public static DriverResilienceOptions ParseOrDefaults(
|
||||||
|
DriverTier tier,
|
||||||
|
string? resilienceConfigJson,
|
||||||
|
IReadOnlyDictionary<DriverCapability, CapabilityPolicy>? typeDefaults,
|
||||||
out string? parseDiagnostic)
|
out string? parseDiagnostic)
|
||||||
{
|
{
|
||||||
parseDiagnostic = null;
|
parseDiagnostic = null;
|
||||||
var baseDefaults = DriverResilienceOptions.GetTierDefaults(tier);
|
IReadOnlyDictionary<DriverCapability, CapabilityPolicy> baseDefaults = DriverResilienceOptions.GetTierDefaults(tier);
|
||||||
|
if (typeDefaults is { Count: > 0 })
|
||||||
|
{
|
||||||
|
var layered = new Dictionary<DriverCapability, CapabilityPolicy>(baseDefaults);
|
||||||
|
foreach (var (capability, policy) in typeDefaults) layered[capability] = policy;
|
||||||
|
baseDefaults = layered;
|
||||||
|
}
|
||||||
var baseOptions = new DriverResilienceOptions { Tier = tier, CapabilityPolicies = baseDefaults };
|
var baseOptions = new DriverResilienceOptions { Tier = tier, CapabilityPolicies = baseDefaults };
|
||||||
|
|
||||||
if (string.IsNullOrWhiteSpace(resilienceConfigJson))
|
if (string.IsNullOrWhiteSpace(resilienceConfigJson))
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Text.Json.Serialization;
|
using System.Text.Json.Serialization;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
@@ -45,9 +46,23 @@ public static class GalaxyDriverFactoryExtensions
|
|||||||
ArgumentNullException.ThrowIfNull(registry);
|
ArgumentNullException.ThrowIfNull(registry);
|
||||||
ArgumentNullException.ThrowIfNull(secretResolver);
|
ArgumentNullException.ThrowIfNull(secretResolver);
|
||||||
registry.Register(DriverTypeName,
|
registry.Register(DriverTypeName,
|
||||||
(id, json) => CreateInstance(id, json, loggerFactory, secretResolver, ackIdentityDirectory));
|
(id, json) => CreateInstance(id, json, loggerFactory, secretResolver, ackIdentityDirectory),
|
||||||
|
capabilityDefaults: CapabilityDefaults);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// #875: Galaxy initialize (gateway session open + the per-template platform seed) took 4–5 s against the
|
||||||
|
/// real VD03 gateway — right at the Tier A 5 s Initialize timeout, so a quarter of restarts failed and paid a
|
||||||
|
/// 10 s retry plus a killed gateway worker. 20 s gives it 4× headroom and stays under the actor's 30 s
|
||||||
|
/// pessimistic bound (<c>DriverInstanceActor.DefaultInitializeTimeout</c>). No retry and breaker threshold 2,
|
||||||
|
/// as in every tier's Initialize policy. A per-instance ResilienceConfig still overrides it.
|
||||||
|
/// </summary>
|
||||||
|
internal static readonly IReadOnlyDictionary<DriverCapability, CapabilityPolicy> CapabilityDefaults =
|
||||||
|
new Dictionary<DriverCapability, CapabilityPolicy>
|
||||||
|
{
|
||||||
|
[DriverCapability.Initialize] = new(TimeoutSeconds: 20, RetryCount: 0, BreakerFailureThreshold: 2),
|
||||||
|
};
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Convenience for tests + standalone callers. Uses the <see cref="NullSecretResolver"/>
|
/// Convenience for tests + standalone callers. Uses the <see cref="NullSecretResolver"/>
|
||||||
/// null-object, so the <c>secret:</c> API-key arm resolves fail-closed (absent) — callers
|
/// null-object, so the <c>secret:</c> API-key arm resolves fail-closed (absent) — callers
|
||||||
|
|||||||
@@ -133,7 +133,8 @@ public static class DriverFactoryBootstrap
|
|||||||
// lmxopcua#777: warn at spawn when a per-instance ResilienceConfig Write timeout exceeds the
|
// lmxopcua#777: warn at spawn when a per-instance ResilienceConfig Write timeout exceeds the
|
||||||
// node's write timeout (it would never take effect). Unregistered ⇒ the default bounds.
|
// node's write timeout (it would never take effect). Unregistered ⇒ the default bounds.
|
||||||
writeTimeoutCeiling: (sp.GetService<ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverWriteBounds>()
|
writeTimeoutCeiling: (sp.GetService<ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverWriteBounds>()
|
||||||
?? ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverWriteBounds.Default).Timeout);
|
?? ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverWriteBounds.Default).Timeout,
|
||||||
|
typeDefaultsResolver: registry.GetCapabilityDefaults);
|
||||||
});
|
});
|
||||||
|
|
||||||
// The tracker's operator-facing reader: a periodic DPS publisher that mirrors each snapshot to the
|
// The tracker's operator-facing reader: a periodic DPS publisher that mirrors each snapshot to the
|
||||||
|
|||||||
+71
@@ -0,0 +1,71 @@
|
|||||||
|
using Shouldly;
|
||||||
|
using Xunit;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Core.Hosting;
|
||||||
|
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// #875: a driver TYPE can register capability defaults that layer over its tier table and under the
|
||||||
|
/// per-instance ResilienceConfig — Galaxy's initialize did not fit Tier A's 5 s.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class DriverTypeCapabilityDefaultsTests
|
||||||
|
{
|
||||||
|
private static readonly Dictionary<DriverCapability, CapabilityPolicy> TypeDefaults = new()
|
||||||
|
{
|
||||||
|
[DriverCapability.Initialize] = new(TimeoutSeconds: 20, RetryCount: 0, BreakerFailureThreshold: 2),
|
||||||
|
};
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Type_defaults_override_the_tier_for_their_capability_only()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, null, TypeDefaults, out var diag);
|
||||||
|
diag.ShouldBeNull();
|
||||||
|
options.Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(20);
|
||||||
|
options.Resolve(DriverCapability.Read).ShouldBe(DriverResilienceOptions.GetTierDefaults(DriverTier.A)[DriverCapability.Read]);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void A_per_instance_ResilienceConfig_still_overrides_the_type_default()
|
||||||
|
{
|
||||||
|
var options = DriverResilienceOptionsParser.ParseOrDefaults(
|
||||||
|
DriverTier.A, """{"capabilityPolicies":{"Initialize":{"timeoutSeconds":12}}}""", TypeDefaults, out _);
|
||||||
|
options.Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(12);
|
||||||
|
options.Resolve(DriverCapability.Initialize).BreakerFailureThreshold.ShouldBe(2, "unspecified fields come from the type default");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Without_type_defaults_the_tier_table_is_unchanged()
|
||||||
|
{
|
||||||
|
DriverResilienceOptionsParser.ParseOrDefaults(DriverTier.A, null, null, out _)
|
||||||
|
.Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(5);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void The_registry_returns_what_a_type_registered_and_null_otherwise()
|
||||||
|
{
|
||||||
|
var registry = new DriverFactoryRegistry();
|
||||||
|
registry.Register("WithDefaults", (_, _) => null!, capabilityDefaults: TypeDefaults);
|
||||||
|
registry.Register("Plain", (_, _) => null!);
|
||||||
|
registry.GetCapabilityDefaults("withdefaults")!.ShouldContainKey(DriverCapability.Initialize);
|
||||||
|
registry.GetCapabilityDefaults("Plain").ShouldBeNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void The_invoker_factory_applies_the_resolved_type_defaults()
|
||||||
|
{
|
||||||
|
var registry = new DriverFactoryRegistry();
|
||||||
|
registry.Register("Slow", (_, _) => null!, capabilityDefaults: TypeDefaults);
|
||||||
|
var tracker = new DriverResilienceStatusTracker();
|
||||||
|
var factory = new DriverCapabilityInvokerFactory(new DriverResiliencePipelineBuilder(statusTracker: tracker), tracker,
|
||||||
|
registry.GetTier, typeDefaultsResolver: registry.GetCapabilityDefaults);
|
||||||
|
var invoker = factory.Create("slow-1", "Slow");
|
||||||
|
// The options the invoker's pipelines are built from (private accessor; no public surface exposes them).
|
||||||
|
var accessor = (Func<DriverResilienceOptions>)invoker.GetType()
|
||||||
|
.GetField("_optionsAccessor", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!
|
||||||
|
.GetValue(invoker)!;
|
||||||
|
accessor().Resolve(DriverCapability.Initialize).TimeoutSeconds.ShouldBe(20);
|
||||||
|
accessor().Resolve(DriverCapability.Read).TimeoutSeconds.ShouldBe(2);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -351,4 +351,15 @@ public sealed class GalaxyDriverFactoryTests
|
|||||||
var driver = GalaxyDriverFactoryExtensions.CreateInstance("g", MinimalConfig);
|
var driver = GalaxyDriverFactoryExtensions.CreateInstance("g", MinimalConfig);
|
||||||
driver.DriverType.ShouldBe("GalaxyMxGateway");
|
driver.DriverType.ShouldBe("GalaxyMxGateway");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>#875: the Galaxy registration carries a 20 s Initialize budget over Tier A's 5 s.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void Register_carries_the_Galaxy_initialize_budget()
|
||||||
|
{
|
||||||
|
var registry = new ZB.MOM.WW.OtOpcUa.Core.Hosting.DriverFactoryRegistry();
|
||||||
|
GalaxyDriverFactoryExtensions.Register(registry, ZB.MOM.WW.OtOpcUa.Driver.Galaxy.NullSecretResolver.Instance);
|
||||||
|
var defaults = registry.GetCapabilityDefaults(GalaxyDriverFactoryExtensions.DriverTypeName);
|
||||||
|
defaults.ShouldNotBeNull();
|
||||||
|
defaults![ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverCapability.Initialize].TimeoutSeconds.ShouldBe(20);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user