feat(telemetry): export the site script compile-cache eviction counter as a Meter (ScadaBridge#68)

SiteScriptCompileCache now records each eviction sweep on scadabridge.site.script_compile_cache.evictions
(ScadaBridgeTelemetry). It rides the existing ZB.MOM.WW.ScadaBridge meter, which is already in
SiteServiceRegistration.ObservedMeters, so no new allowlist entry is needed; a Host test pins that
the meter stays allowlisted.
This commit is contained in:
Joseph Doherty
2026-11-09 12:08:25 -05:00
parent e2b15bd520
commit 79957c4b96
4 changed files with 59 additions and 0 deletions
@@ -76,6 +76,17 @@ public static class ScadaBridgeTelemetry
Meter.CreateCounter<long>("scadabridge.site.alarm_cache.buffer_dropped", unit: "1",
description: "Live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer (drop-oldest).");
/// <summary>
/// Incremented by the number of entries the site script compile cache evicts when it crosses its
/// bound (ScadaBridge#68). Eviction does not unload the Roslyn assembly, so each recompile of an
/// evicted body loads another permanent copy: a sustained climb means compile-cache thrash and
/// monotonic working-set growth - raise <c>ScadaBridge:SiteRuntime:ScriptCompileCacheMaxEntries</c>.
/// Alert on this instead of the rate-limited Warning log.
/// </summary>
private static readonly Counter<long> _scriptCompileCacheEvictions =
Meter.CreateCounter<long>("scadabridge.site.script_compile_cache.evictions", unit: "1",
description: "Entries evicted from the site script compile cache at its bound (each recompile of an evicted body loads another permanent Roslyn assembly).");
/// <summary>
/// Incremented for each debug event evicted from a central debug session's bounded
/// pre-snapshot buffer (drop-oldest). The Debug View is lossy-under-backpressure
@@ -370,6 +381,10 @@ public static class ScadaBridgeTelemetry
/// <param name="count">Number of deltas evicted.</param>
public static void RecordLiveAlarmBufferDrop(long count = 1) => _liveAlarmBufferDrops.Add(count);
/// <summary>Records entries evicted from the site script compile cache.</summary>
/// <param name="count">Number of entries evicted by one sweep.</param>
public static void RecordScriptCompileCacheEvictions(long count) => _scriptCompileCacheEvictions.Add(count);
/// <summary>Records debug events evicted from a debug session's bounded pre-snapshot buffer.</summary>
/// <param name="count">Number of events evicted.</param>
public static void RecordDebugPreSnapshotDrop(long count = 1) => _debugPreSnapshotDrops.Add(count);
@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
using System.Security.Cryptography;
using System.Text;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
@@ -105,6 +106,8 @@ internal sealed class SiteScriptCompileCache
/// <c>Script&lt;object?&gt;</c>; the assembly Roslyn loaded for it stays in the default load context
/// for the process lifetime, and a later miss for the same body loads a second copy. A rising value
/// therefore means monotonic working-set growth — raise <c>ScriptCompileCacheMaxEntries</c>.
/// Also exported as the <c>scadabridge.site.script_compile_cache.evictions</c> counter
/// (<see cref="ScadaBridgeTelemetry.RecordScriptCompileCacheEvictions"/>) for Prometheus alerting.
/// </summary>
public long Evictions => Interlocked.Read(ref _evictions);
@@ -183,6 +186,8 @@ internal sealed class SiteScriptCompileCache
_cache.TryRemove(victim, out _);
Interlocked.Add(ref _evictions, victims.Length);
// Exported on the shared ScadaBridge meter (already in SiteServiceRegistration.ObservedMeters).
ScadaBridgeTelemetry.RecordScriptCompileCacheEvictions(victims.Length);
WarnEvicted();
}
}
@@ -245,6 +245,17 @@ public class SiteLocalDbWiringTests : IDisposable
Assert.Contains(LocalDbMetrics.MeterName, SiteServiceRegistration.ObservedMeters);
}
/// <summary>Verifies that the ScadaBridge application meter (carrying the compile-cache eviction counter) is exported.</summary>
[Fact]
public void Site_Telemetry_Allowlists_TheScadaBridgeMeter()
{
// ScadaBridge#68: scadabridge.site.script_compile_cache.evictions lives on this meter; an
// unlisted meter is dropped silently (ZbTelemetryOptions.Meters is an allowlist).
Assert.Contains(
ZB.MOM.WW.ScadaBridge.Commons.Observability.ScadaBridgeTelemetry.MeterName,
SiteServiceRegistration.ObservedMeters);
}
/// <summary>Verifies that the Phase 2 tables are created in the consolidated LocalDb file.</summary>
[Fact]
public void Site_LocalDb_CreatesThePhase2Tables_InTheConsolidatedFile()
@@ -253,4 +253,32 @@ public class SiteScriptCompileCacheTests
Assert.True(cache.Count <= 16);
Assert.True(cache.Evictions > 0);
}
/// <summary>
/// ScadaBridge#68: evictions are exported as a Meter counter (not only the rate-limited Warning
/// log), on the ScadaBridge meter that the Host already allowlists for export.
/// </summary>
[Fact]
public void Eviction_IsExportedAsAMeterCounter()
{
long exported = 0;
using var listener = new System.Diagnostics.Metrics.MeterListener();
listener.InstrumentPublished = (instrument, l) =>
{
if (instrument.Meter.Name == ZB.MOM.WW.ScadaBridge.Commons.Observability.ScadaBridgeTelemetry.MeterName
&& instrument.Name == "scadabridge.site.script_compile_cache.evictions")
l.EnableMeasurementEvents(instrument);
};
listener.SetMeasurementEventCallback<long>((_, measurement, _, _) => Interlocked.Add(ref exported, measurement));
listener.Start();
var cache = new SiteScriptCompileCache(16, logger: null);
for (var i = 0; i < 40; i++)
cache.GetOrAdd("return " + i + ";", typeof(ScriptGlobals), Ok);
// The instrument is process-wide, so a parallel test's evictions may add to it: bound from below.
Assert.True(cache.Evictions > 0);
Assert.True(Interlocked.Read(ref exported) >= cache.Evictions,
$"exported {exported} but the cache evicted {cache.Evictions}");
}
}