using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.Transport.Import;
namespace ZB.MOM.WW.ScadaBridge.Transport.IntegrationTests.Import;
///
/// Register row 52: the bundle-unlock throttle's counters used to be per-node AND
/// per-process, so the effective fleet cap was N x MaxUnlockAttemptsPerIpPerHour
/// per client and a node restart silently cleared the window. These tests stand up TWO
/// independent limiter instances — the stand-in for central-a and central-b — over ONE
/// shared relational store and prove they enforce a single combined window, plus the
/// fail-open behaviour a store outage must produce.
///
public sealed class FleetWideUnlockThrottleTests : IDisposable
{
private readonly SqliteConnection _connection;
private readonly ServiceProvider _provider;
/// Initializes a new instance of the FleetWideUnlockThrottleTests class.
public FleetWideUnlockThrottleTests()
{
// One open in-memory SQLite connection for the fixture's lifetime — the database
// lives exactly as long as the connection, and both "nodes" resolve scoped
// contexts over it, which is what makes the store genuinely shared.
_connection = new SqliteConnection("DataSource=:memory:");
_connection.Open();
var services = new ServiceCollection();
services.AddDbContext(opts => opts.UseSqlite(_connection));
services.AddScoped();
_provider = services.BuildServiceProvider();
using var scope = _provider.CreateScope();
scope.ServiceProvider.GetRequiredService().Database.EnsureCreated();
}
/// Disposes the service provider and the shared connection.
public void Dispose()
{
_provider.Dispose();
_connection.Dispose();
}
///
/// Builds a limiter over the shared store, standing in for one central node's
/// singleton. Each call returns a fully independent instance (its own process-local
/// fallback buckets, its own prune claim) — exactly the isolation the two real nodes
/// have.
///
/// Clock the node observes.
/// A limiter whose window is the shared store.
private BundleUnlockRateLimiter NewNode(TimeProvider clock) => new(
clock,
BundleUnlockRateLimiter.DefaultWindow,
new SqlBundleUnlockAttemptWindow(
_provider.GetRequiredService(), clock),
logger: null);
/// Verifies that two nodes sharing one store enforce a single combined throttle window.
/// A task that represents the asynchronous operation.
[Fact]
public async Task TwoNodesSharingOneStore_EnforceASingleCombinedWindow()
{
// The headline contract: attempts spread across both central nodes count
// against ONE budget. Before this, five attempts per node were both allowed.
var clock = new TestClock(new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero));
var nodeA = NewNode(clock);
var nodeB = NewNode(clock);
for (var i = 0; i < 5; i++)
{
Assert.True(await nodeA.TryRegisterAttemptAsync("10.0.0.1", 10));
Assert.True(await nodeB.TryRegisterAttemptAsync("10.0.0.1", 10));
}
// The combined budget of 10 is now spent — on EITHER node.
Assert.False(await nodeA.TryRegisterAttemptAsync("10.0.0.1", 10));
Assert.False(await nodeB.TryRegisterAttemptAsync("10.0.0.1", 10));
}
/// Verifies that denied attempts do not consume any of the throttle budget.
/// A task that represents the asynchronous operation.
[Fact]
public async Task DeniedAttempts_BurnNoBudget()
{
// Register row 44's contract, preserved through the move to the shared store: a
// rejected attempt is not recorded, so hammering cannot roll the window forward
// and extend the client's own lockout.
var clock = new TestClock(new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero));
var nodeA = NewNode(clock);
for (var i = 0; i < 3; i++)
{
Assert.True(await nodeA.TryRegisterAttemptAsync("10.0.0.1", 3));
}
for (var i = 0; i < 5; i++)
{
Assert.False(await nodeA.TryRegisterAttemptAsync("10.0.0.1", 3));
}
// Exactly the 3 permitted attempts were persisted — the 5 denials added nothing.
using var scope = _provider.CreateScope();
var repo = scope.ServiceProvider.GetRequiredService();
var recorded = await repo.CountAttemptsSinceAsync(
"10.0.0.1", clock.GetUtcNow().UtcDateTime.AddHours(-1));
Assert.Equal(3, recorded);
}
/// Verifies that the throttle window survives a restart.
/// A task that represents the asynchronous operation.
[Fact]
public async Task TheWindowIsRestartDurable()
{
// A node restart used to clear the window outright. A brand-new limiter instance
// (the restarted process) must still see the budget the old one spent.
var clock = new TestClock(new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero));
var before = NewNode(clock);
for (var i = 0; i < 3; i++)
{
Assert.True(await before.TryRegisterAttemptAsync("10.0.0.1", 3));
}
var afterRestart = NewNode(clock);
Assert.False(await afterRestart.TryRegisterAttemptAsync("10.0.0.1", 3));
}
/// Verifies that the throttle window slides so an aged-out budget is released.
/// A task that represents the asynchronous operation.
[Fact]
public async Task TheWindowSlides_SoAnAgedOutBudgetIsReleased()
{
var clock = new TestClock(new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero));
var node = NewNode(clock);
for (var i = 0; i < 3; i++)
{
Assert.True(await node.TryRegisterAttemptAsync("10.0.0.1", 3));
}
Assert.False(await node.TryRegisterAttemptAsync("10.0.0.1", 3));
clock.Advance(TimeSpan.FromMinutes(61));
Assert.True(await node.TryRegisterAttemptAsync("10.0.0.1", 3));
}
/// Verifies that throttle buckets are isolated per client.
/// A task that represents the asynchronous operation.
[Fact]
public async Task BucketsAreIsolatedPerClient()
{
var clock = new TestClock(new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero));
var node = NewNode(clock);
Assert.True(await node.TryRegisterAttemptAsync("10.0.0.1", 1));
Assert.False(await node.TryRegisterAttemptAsync("10.0.0.1", 1));
// A different client — and the shared unattributed bucket — are unaffected.
Assert.True(await node.TryRegisterAttemptAsync("10.0.0.2", 1));
Assert.True(await node.TryRegisterAttemptAsync("", 1));
}
/// Verifies that a store failure fails open and does not block the unlock.
/// A task that represents the asynchronous operation.
[Fact]
public async Task StoreFailure_FailsOpen_AndDoesNotBlockTheUnlock()
{
// Availability of legitimate imports outranks a best-effort rate cap: a database
// error must permit the attempt, not take the import surface down with the DB.
var window = Substitute.For();
window
.TryRegisterAttemptAsync(
Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any())
.ThrowsAsync(new InvalidOperationException("database unavailable"));
var limiter = new BundleUnlockRateLimiter(
TimeProvider.System, BundleUnlockRateLimiter.DefaultWindow, window, logger: null);
for (var i = 0; i < 20; i++)
{
Assert.True(await limiter.TryRegisterAttemptAsync("10.0.0.1", 1));
}
}
/// Verifies that caller cancellation is not swallowed by the fail-open path.
/// A task that represents the asynchronous operation.
[Fact]
public async Task CallerCancellation_IsNotSwallowedByTheFailOpenPath()
{
// Fail-open must not hide the caller's own cancellation — that would turn an
// aborted request into a silently permitted attempt.
var window = Substitute.For();
window
.TryRegisterAttemptAsync(
Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any())
.ThrowsAsync(new OperationCanceledException());
var limiter = new BundleUnlockRateLimiter(
TimeProvider.System, BundleUnlockRateLimiter.DefaultWindow, window, logger: null);
using var cts = new CancellationTokenSource();
await cts.CancelAsync();
await Assert.ThrowsAnyAsync(
() => limiter.TryRegisterAttemptAsync("10.0.0.1", 10, cts.Token));
}
/// Verifies that without a shared window the limiter still enforces a process-local cap.
/// A task that represents the asynchronous operation.
[Fact]
public async Task WithoutASharedWindow_TheLimiterStillEnforcesAProcessLocalCap()
{
// Hosts that register Transport without a shared window keep the pre-shared-window
// behaviour rather than losing the cap entirely.
var clock = new TestClock(new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero));
var limiter = new BundleUnlockRateLimiter(clock, BundleUnlockRateLimiter.DefaultWindow);
Assert.False(limiter.IsFleetWide);
Assert.True(await limiter.TryRegisterAttemptAsync("10.0.0.1", 1));
Assert.False(await limiter.TryRegisterAttemptAsync("10.0.0.1", 1));
}
/// Verifies that aged rows are pruned opportunistically once the prune interval has elapsed.
/// A task that represents the asynchronous operation.
[Fact]
public async Task AgedRowsArePrunedOpportunistically_OnceThePruneIntervalHasElapsed()
{
// No background service: the prune rides the unlock path, claimed at most once per
// PruneMinInterval. Rows are retained for PruneRetentionWindows x window so a prune
// racing a count can never delete a row the count still needs.
var clock = new TestClock(new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero));
var node = NewNode(clock);
Assert.True(await node.TryRegisterAttemptAsync("10.0.0.1", 10));
// Move past both the retention horizon (2 hours) and the prune cadence.
clock.Advance(TimeSpan.FromHours(3));
Assert.True(await node.TryRegisterAttemptAsync("10.0.0.2", 10));
using var scope = _provider.CreateScope();
var ctx = scope.ServiceProvider.GetRequiredService();
var remaining = await ctx.BundleUnlockAttempts.AsNoTracking().ToListAsync();
// The aged row is gone; the one just written by the pruning attempt survives.
Assert.Equal("10.0.0.2", Assert.Single(remaining).ClientKey);
}
/// Verifies that recent rows survive a prune pass.
/// A task that represents the asynchronous operation.
[Fact]
public async Task RecentRowsSurviveAPrunePass()
{
// Negative control for the prune horizon: an attempt still inside the retention
// window must not be deleted, or the throttle would forget live budget.
var clock = new TestClock(new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero));
var node = NewNode(clock);
Assert.True(await node.TryRegisterAttemptAsync("10.0.0.1", 10));
// Past the prune cadence (15 min) but inside the 2-hour retention horizon.
clock.Advance(TimeSpan.FromMinutes(30));
Assert.True(await node.TryRegisterAttemptAsync("10.0.0.2", 10));
using var scope = _provider.CreateScope();
var ctx = scope.ServiceProvider.GetRequiredService();
Assert.Equal(2, await ctx.BundleUnlockAttempts.AsNoTracking().CountAsync());
}
/// Verifies that the scheduled prune clears aged rows even when there is no unlock traffic.
/// A task that represents the asynchronous operation.
[Fact]
public async Task ScheduledPrune_ClearsAgedRows_WithNoUnlockTrafficAtAll()
{
// Register row 65: the prune used to ride the unlock path ONLY, so a cluster that
// stopped importing entirely left its last rows in place indefinitely. The
// periodic sweep now drives PruneAsync directly — no second unlock attempt is made
// anywhere in this test, which is the whole point.
var clock = new TestClock(new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero));
var window = new SqlBundleUnlockAttemptWindow(
_provider.GetRequiredService(), clock);
Assert.True(await window.TryRegisterAttemptAsync(
"10.0.0.1", 10, BundleUnlockRateLimiter.DefaultWindow));
// Past both the 2-hour retention horizon and the 15-minute prune cadence.
clock.Advance(TimeSpan.FromHours(3));
await window.PruneAsync(BundleUnlockRateLimiter.DefaultWindow);
using var scope = _provider.CreateScope();
var ctx = scope.ServiceProvider.GetRequiredService();
Assert.Empty(await ctx.BundleUnlockAttempts.AsNoTracking().ToListAsync());
}
/// Verifies that the scheduled prune respects the retention horizon and the cadence claim.
/// A task that represents the asynchronous operation.
[Fact]
public async Task ScheduledPrune_RespectsTheRetentionHorizonAndTheCadenceClaim()
{
// Two negative controls in one: a row inside the retention horizon survives, and
// the shared cadence claim means a burst of ticks cannot prune more often than
// PruneMinInterval (so ticking on the 1-minute session sweep is cheap).
var clock = new TestClock(new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.Zero));
var window = new SqlBundleUnlockAttemptWindow(
_provider.GetRequiredService(), clock);
Assert.True(await window.TryRegisterAttemptAsync(
"10.0.0.1", 10, BundleUnlockRateLimiter.DefaultWindow));
// Recent row, past the cadence: pruning runs but must delete nothing.
clock.Advance(TimeSpan.FromMinutes(30));
for (var i = 0; i < 5; i++)
{
await window.PruneAsync(BundleUnlockRateLimiter.DefaultWindow);
}
using var scope = _provider.CreateScope();
var ctx = scope.ServiceProvider.GetRequiredService();
Assert.Equal(1, await ctx.BundleUnlockAttempts.AsNoTracking().CountAsync());
}
/// Verifies that the scheduled prune does not throw when the store is unavailable.
/// A task that represents the asynchronous operation.
[Fact]
public async Task ScheduledPrune_DoesNotThrow_WhenTheStoreIsUnavailable()
{
// The sweep that drives this has no caller to report to, so PruneAsync's contract
// is "never throw" — a database blip must not kill the background service loop.
var services = new ServiceCollection();
services.AddScoped(
_ => throw new InvalidOperationException("database unavailable"));
using var broken = services.BuildServiceProvider();
var window = new SqlBundleUnlockAttemptWindow(
broken.GetRequiredService(), TimeProvider.System);
await window.PruneAsync(BundleUnlockRateLimiter.DefaultWindow);
}
/// Minimal mutable-clock for deterministic windows.
private sealed class TestClock : TimeProvider
{
private DateTimeOffset _now;
/// Initializes a new instance of the TestClock class.
/// The initial time of the clock.
public TestClock(DateTimeOffset start) { _now = start; }
///
public override DateTimeOffset GetUtcNow() => _now;
/// Moves the test clock forward by the given amount.
/// The amount to advance the clock by.
public void Advance(TimeSpan by) => _now = _now.Add(by);
}
}