fix(communication): evict idle SiteStreamGrpcClientFactory (site, endpoint) clients (ScadaBridge#67)

Each cached client records when it was last handed out; a periodic sweep (half of the new
CommunicationOptions.GrpcClientIdleTtl, default 30 min, 0 disables) disposes and removes entries
unused for the TTL that have no live subscription (SiteStreamGrpcClient.ActiveSubscriptionCount).
Lookup and the sweep's check-and-remove share one lock so a handed-out client is never disposed
under its caller; an evicted client is re-created on next GetOrCreate.
This commit is contained in:
Joseph Doherty
2026-11-09 12:07:28 -05:00
parent e1734675e9
commit e2b15bd520
6 changed files with 300 additions and 14 deletions
@@ -0,0 +1,162 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
/// <summary>
/// ScadaBridge#67: <see cref="SiteStreamGrpcClientFactory"/> evicts (and disposes) clients whose
/// <c>(site, endpoint)</c> has gone unused for <see cref="CommunicationOptions.GrpcClientIdleTtl"/>,
/// so a renamed or removed address does not leave its channel resident for the life of the process.
/// </summary>
public class SiteStreamGrpcClientFactoryIdleEvictionTests
{
private static readonly TimeSpan Ttl = TimeSpan.FromMinutes(10);
/// <summary>Test clock: only advances when told to.</summary>
private sealed class ManualTimeProvider : TimeProvider
{
private long _ticks = TimeSpan.FromHours(1).Ticks;
public override long TimestampFrequency => TimeSpan.TicksPerSecond;
public override long GetTimestamp() => Interlocked.Read(ref _ticks);
public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks);
}
private sealed class TrackingClient(string endpoint) : SiteStreamGrpcClient(endpoint)
{
public int DisposeCount;
public override void Dispose() { Interlocked.Increment(ref DisposeCount); base.Dispose(); }
public override ValueTask DisposeAsync() { Interlocked.Increment(ref DisposeCount); return base.DisposeAsync(); }
}
private sealed class Factory(ManualTimeProvider clock, TimeSpan ttl) : SiteStreamGrpcClientFactory(
NullLoggerFactory.Instance,
Options.Create(new CommunicationOptions { GrpcClientIdleTtl = ttl }),
pskProvider: null,
clock)
{
public int Created;
protected override SiteStreamGrpcClient CreateClient(string siteIdentifier, string grpcEndpoint)
{
Interlocked.Increment(ref Created);
return new TrackingClient(grpcEndpoint);
}
}
/// <summary>An unused client is disposed and removed once the TTL elapses; a fresh one is created on next use.</summary>
[Fact]
public void IdleClient_IsEvictedAndDisposed_ThenRecreatedOnDemand()
{
var clock = new ManualTimeProvider();
using var factory = new Factory(clock, Ttl);
var old = (TrackingClient)factory.GetOrCreate("site-a", "http://old:5100");
clock.Advance(Ttl + TimeSpan.FromSeconds(1));
Assert.Equal(1, factory.EvictIdle());
Assert.Equal(1, old.DisposeCount);
Assert.Null(factory.TryGet("site-a", "http://old:5100"));
var fresh = factory.GetOrCreate("site-a", "http://old:5100");
Assert.NotSame(old, fresh);
}
/// <summary>A client used within the TTL survives the sweep - and using it resets the idle clock.</summary>
[Fact]
public void RecentlyUsedClient_IsNotEvicted()
{
var clock = new ManualTimeProvider();
using var factory = new Factory(clock, Ttl);
var kept = (TrackingClient)factory.GetOrCreate("site-a", "http://a:5100");
var idle = (TrackingClient)factory.GetOrCreate("site-a", "http://b:5100");
clock.Advance(TimeSpan.FromMinutes(8));
Assert.Same(kept, factory.GetOrCreate("site-a", "http://a:5100")); // touch
clock.Advance(TimeSpan.FromMinutes(8));
Assert.Equal(1, factory.EvictIdle());
Assert.Equal(0, kept.DisposeCount);
Assert.Equal(1, idle.DisposeCount);
Assert.Same(kept, factory.TryGet("site-a", "http://a:5100"));
}
/// <summary>A client with a live subscription is never evicted, however long since it was handed out.</summary>
[Fact]
public void ClientWithActiveSubscription_IsNotEvicted()
{
var clock = new ManualTimeProvider();
using var factory = new Factory(clock, Ttl);
var busy = (TrackingClient)factory.GetOrCreate("site-a", "http://a:5100");
using var cts = new CancellationTokenSource();
busy.RegisterSubscription("corr-1", cts);
clock.Advance(Ttl * 3);
Assert.Equal(0, factory.EvictIdle());
Assert.Equal(0, busy.DisposeCount);
// Once the subscription is gone it becomes evictable again.
busy.Unsubscribe("corr-1");
Assert.Equal(1, factory.EvictIdle());
Assert.Equal(1, busy.DisposeCount);
}
/// <summary>A zero TTL disables idle eviction entirely.</summary>
[Fact]
public void ZeroTtl_DisablesEviction()
{
var clock = new ManualTimeProvider();
using var factory = new Factory(clock, TimeSpan.Zero);
var client = (TrackingClient)factory.GetOrCreate("site-a", "http://a:5100");
clock.Advance(TimeSpan.FromDays(30));
Assert.Equal(0, factory.EvictIdle());
Assert.Equal(0, client.DisposeCount);
}
/// <summary>Hand-out racing the sweep never throws and never returns null; evicted entries are re-created.</summary>
[Fact]
public async Task ConcurrentGetOrCreateAndEviction_IsThreadSafe()
{
var clock = new ManualTimeProvider();
using var factory = new Factory(clock, TimeSpan.FromSeconds(1));
var stop = false;
var evictor = Task.Run(() =>
{
while (!Volatile.Read(ref stop))
{
clock.Advance(TimeSpan.FromSeconds(2));
factory.EvictIdle();
}
});
var getters = Enumerable.Range(0, 4).Select(_ => Task.Run(() =>
{
for (var i = 0; i < 20_000; i++)
{
Assert.NotNull(factory.GetOrCreate("site-a", "http://a:5100"));
factory.TryGet("site-a", "http://a:5100");
}
})).ToArray();
await Task.WhenAll(getters);
Volatile.Write(ref stop, true);
await evictor;
Assert.True(factory.Created >= 1);
}
/// <summary>The validator rejects a negative idle TTL and accepts zero.</summary>
[Fact]
public void Validator_RejectsNegativeIdleTtl()
{
var validator = new CommunicationOptionsValidator();
var bad = validator.Validate(null, new CommunicationOptions { GrpcClientIdleTtl = TimeSpan.FromSeconds(-1) });
var zero = validator.Validate(null, new CommunicationOptions { GrpcClientIdleTtl = TimeSpan.Zero });
Assert.True(bad.Failed);
Assert.Contains("GrpcClientIdleTtl", string.Join(" ", bad.Failures!));
Assert.DoesNotContain("GrpcClientIdleTtl", string.Join(" ", zero.Failures ?? []));
}
}