feat(transport): counter for import-session cap rejections (register row 66)
Row 53 shipped the scadabridge.central.transport.import_sessions.open gauge
and a Warning log on a cap hit, but a log line is not something an operator
can alert on. Adds the companion counter
scadabridge.central.transport.import_sessions.rejected
on the existing ZB.MOM.WW.ScadaBridge meter, so no ObservedMeters allowlist
change is needed (an unlisted meter exports nothing, silently).
The emit point is deliberately narrow. BundleSessionStore.Open tests the cap
TWICE — once on entry and again after EvictExpired() — and only the second
test is a rejection: a store full of merely expired sessions reclaims a slot
and admits the caller. Counting the first would make the series fire
routinely on any node whose sweep had not yet run, turning a page-worthy
signal into background noise. Re-Opening an already-open session id skips
the cap block entirely and is likewise not counted.
The two instruments are diagnostic as a pair: rejections rising while the
gauge oscillates below the cap is genuine concurrency (raise the cap);
rejections rising while the gauge sits pinned at the cap is a session leak
(register row 43's signature), where raising the cap only defers the problem.
Tests assert a delta over a scoped action rather than an absolute value, so
they do not inherit the register row 32 static-state flake that makes
QueueDepthGaugeTests fail only under the full concurrent suite. Negative
controls cover under-cap, expired-reclaim, and duplicate-session-id.
Tests: 11 passed in BundleSessionObservabilityTests (6 existing + 5 new).
This commit is contained in:
Joseph Doherty
+136
@@ -19,6 +19,7 @@ namespace ZB.MOM.WW.ScadaBridge.Transport.Tests.Import;
|
||||
public sealed class BundleSessionObservabilityTests
|
||||
{
|
||||
private const string OpenSessionsGauge = "scadabridge.central.transport.import_sessions.open";
|
||||
private const string RejectedSessionsCounter = "scadabridge.central.transport.import_sessions.rejected";
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Warning log on cap rejection
|
||||
@@ -83,6 +84,108 @@ public sealed class BundleSessionObservabilityTests
|
||||
Assert.Empty(logger.Entries);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Rejection counter (register row 66)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Open_AtCap_IncrementsTheRejectionCounter()
|
||||
{
|
||||
// Row 53 shipped the gauge + Warning; a log line is not alertable. This counter is
|
||||
// the series an operator can page on, so it must move on a genuine cap hit.
|
||||
var clock = new TestClock(DateTimeOffset.UtcNow);
|
||||
var opts = Options.Create(new TransportOptions { MaxConcurrentImportSessions = 3 });
|
||||
var sut = new BundleSessionStore(opts, clock, NullLogger<BundleSessionStore>.Instance);
|
||||
var future = clock.GetUtcNow().AddMinutes(30);
|
||||
|
||||
for (var i = 0; i < 3; i++)
|
||||
{
|
||||
sut.Open(SessionExpiringAt(future));
|
||||
}
|
||||
|
||||
var delta = ReadRejectionCounter(() =>
|
||||
Assert.Throws<InvalidOperationException>(() => sut.Open(SessionExpiringAt(future))));
|
||||
|
||||
Assert.Equal(1, delta);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Open_RepeatedCapHits_AccumulateOnTheRejectionCounter()
|
||||
{
|
||||
// A counter that saturated at 1 would hide exactly the case it exists for: a
|
||||
// sustained rejection rate is what distinguishes real contention from a one-off.
|
||||
var clock = new TestClock(DateTimeOffset.UtcNow);
|
||||
var opts = Options.Create(new TransportOptions { MaxConcurrentImportSessions = 1 });
|
||||
var sut = new BundleSessionStore(opts, clock, NullLogger<BundleSessionStore>.Instance);
|
||||
var future = clock.GetUtcNow().AddMinutes(30);
|
||||
sut.Open(SessionExpiringAt(future));
|
||||
|
||||
var delta = ReadRejectionCounter(() =>
|
||||
{
|
||||
for (var i = 0; i < 3; i++)
|
||||
{
|
||||
Assert.Throws<InvalidOperationException>(() => sut.Open(SessionExpiringAt(future)));
|
||||
}
|
||||
});
|
||||
|
||||
Assert.Equal(3, delta);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Open_UnderCap_DoesNotIncrementTheRejectionCounter()
|
||||
{
|
||||
// Negative control: the counter is a rejection signal, not an open-session signal.
|
||||
var clock = new TestClock(DateTimeOffset.UtcNow);
|
||||
var opts = Options.Create(new TransportOptions { MaxConcurrentImportSessions = 3 });
|
||||
var sut = new BundleSessionStore(opts, clock, NullLogger<BundleSessionStore>.Instance);
|
||||
|
||||
var delta = ReadRejectionCounter(() =>
|
||||
sut.Open(SessionExpiringAt(clock.GetUtcNow().AddMinutes(30))));
|
||||
|
||||
Assert.Equal(0, delta);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Open_AtCapOfExpiredSessions_ReclaimsWithoutIncrementingTheRejectionCounter()
|
||||
{
|
||||
// The load-bearing negative control. The cap check runs TWICE — once before
|
||||
// EvictExpired and once after — and only the post-eviction check is a rejection.
|
||||
// Counting the first would make the series fire routinely on any node whose sweep
|
||||
// has not yet run, turning a page-worthy signal into background noise.
|
||||
var clock = new TestClock(DateTimeOffset.UtcNow);
|
||||
var opts = Options.Create(new TransportOptions { MaxConcurrentImportSessions = 2 });
|
||||
var sut = new BundleSessionStore(opts, clock, NullLogger<BundleSessionStore>.Instance);
|
||||
|
||||
sut.Open(SessionExpiringAt(clock.GetUtcNow().AddMinutes(5)));
|
||||
sut.Open(SessionExpiringAt(clock.GetUtcNow().AddMinutes(5)));
|
||||
clock.Advance(TimeSpan.FromMinutes(10));
|
||||
|
||||
var delta = ReadRejectionCounter(() =>
|
||||
sut.Open(SessionExpiringAt(clock.GetUtcNow().AddMinutes(30))));
|
||||
|
||||
Assert.Equal(0, delta);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Open_ReopeningAnExistingSessionIdAtCap_DoesNotIncrementTheRejectionCounter()
|
||||
{
|
||||
// Re-Opening an id already in the store skips the cap block entirely and overwrites
|
||||
// in place. It consumes no new slot and turns nobody away, so it must not count.
|
||||
var clock = new TestClock(DateTimeOffset.UtcNow);
|
||||
var opts = Options.Create(new TransportOptions { MaxConcurrentImportSessions = 2 });
|
||||
var sut = new BundleSessionStore(opts, clock, NullLogger<BundleSessionStore>.Instance);
|
||||
var future = clock.GetUtcNow().AddMinutes(30);
|
||||
|
||||
var first = SessionExpiringAt(future);
|
||||
sut.Open(first);
|
||||
sut.Open(SessionExpiringAt(future));
|
||||
|
||||
// Store is now exactly at the cap of 2.
|
||||
var delta = ReadRejectionCounter(() => sut.Open(first));
|
||||
|
||||
Assert.Equal(0, delta);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// OpenSessionCount / gauge
|
||||
// ------------------------------------------------------------------
|
||||
@@ -197,6 +300,39 @@ public sealed class BundleSessionObservabilityTests
|
||||
return observed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs <paramref name="act"/> with a listener attached to the import-session rejection
|
||||
/// counter and returns the number of rejections recorded DURING that action.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately a delta over a scoped action rather than an absolute read. The counter
|
||||
/// is a process-global static, so an absolute assertion would inherit the register
|
||||
/// row 32 flake that makes <c>QueueDepthGaugeTests</c> fail only under the full
|
||||
/// concurrent suite. A counter measured this way is immune: the listener exists only
|
||||
/// for the duration of <paramref name="act"/>, and every increment is driven
|
||||
/// synchronously by this test's own <c>Open</c> call on its own store instance, so a
|
||||
/// concurrently-running test class cannot contribute a measurement to this window.
|
||||
/// </remarks>
|
||||
/// <param name="act">The action to measure.</param>
|
||||
/// <returns>Rejections counted while <paramref name="act"/> ran.</returns>
|
||||
private static long ReadRejectionCounter(Action act)
|
||||
{
|
||||
long delta = 0;
|
||||
using var listener = new MeterListener();
|
||||
listener.InstrumentPublished = (instrument, l) =>
|
||||
{
|
||||
if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName
|
||||
&& instrument.Name == RejectedSessionsCounter)
|
||||
{
|
||||
l.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
listener.SetMeasurementEventCallback<long>((_, measurement, _, _) => delta += measurement);
|
||||
listener.Start();
|
||||
act();
|
||||
return delta;
|
||||
}
|
||||
|
||||
private static BundleSession SessionExpiringAt(DateTimeOffset expiresAt) => new()
|
||||
{
|
||||
SessionId = Guid.NewGuid(),
|
||||
|
||||
Reference in New Issue
Block a user