diff --git a/src/ZB.MOM.WW.ScadaBridge.Commons/Observability/ScadaBridgeTelemetry.cs b/src/ZB.MOM.WW.ScadaBridge.Commons/Observability/ScadaBridgeTelemetry.cs index ce653c63..5b4a2a44 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Commons/Observability/ScadaBridgeTelemetry.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Commons/Observability/ScadaBridgeTelemetry.cs @@ -76,6 +76,17 @@ public static class ScadaBridgeTelemetry Meter.CreateCounter("scadabridge.site.alarm_cache.buffer_dropped", unit: "1", description: "Live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer (drop-oldest)."); + /// + /// Incremented by the number of entries the site script compile cache evicts when it crosses its + /// bound (ScadaBridge#68). Eviction does not unload the Roslyn assembly, so each recompile of an + /// evicted body loads another permanent copy: a sustained climb means compile-cache thrash and + /// monotonic working-set growth - raise ScadaBridge:SiteRuntime:ScriptCompileCacheMaxEntries. + /// Alert on this instead of the rate-limited Warning log. + /// + private static readonly Counter _scriptCompileCacheEvictions = + Meter.CreateCounter("scadabridge.site.script_compile_cache.evictions", unit: "1", + description: "Entries evicted from the site script compile cache at its bound (each recompile of an evicted body loads another permanent Roslyn assembly)."); + /// /// Incremented for each debug event evicted from a central debug session's bounded /// pre-snapshot buffer (drop-oldest). The Debug View is lossy-under-backpressure @@ -370,6 +381,10 @@ public static class ScadaBridgeTelemetry /// Number of deltas evicted. public static void RecordLiveAlarmBufferDrop(long count = 1) => _liveAlarmBufferDrops.Add(count); + /// Records entries evicted from the site script compile cache. + /// Number of entries evicted by one sweep. + public static void RecordScriptCompileCacheEvictions(long count) => _scriptCompileCacheEvictions.Add(count); + /// Records debug events evicted from a debug session's bounded pre-snapshot buffer. /// Number of events evicted. public static void RecordDebugPreSnapshotDrop(long count = 1) => _debugPreSnapshotDrops.Add(count); diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/SiteScriptCompileCache.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/SiteScriptCompileCache.cs index f64a9444..75f735f6 100644 --- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/SiteScriptCompileCache.cs +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/SiteScriptCompileCache.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Security.Cryptography; using System.Text; using Microsoft.Extensions.Logging; +using ZB.MOM.WW.ScadaBridge.Commons.Observability; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; @@ -105,6 +106,8 @@ internal sealed class SiteScriptCompileCache /// Script<object?>; the assembly Roslyn loaded for it stays in the default load context /// for the process lifetime, and a later miss for the same body loads a second copy. A rising value /// therefore means monotonic working-set growth — raise ScriptCompileCacheMaxEntries. + /// Also exported as the scadabridge.site.script_compile_cache.evictions counter + /// () for Prometheus alerting. /// public long Evictions => Interlocked.Read(ref _evictions); @@ -183,6 +186,8 @@ internal sealed class SiteScriptCompileCache _cache.TryRemove(victim, out _); Interlocked.Add(ref _evictions, victims.Length); + // Exported on the shared ScadaBridge meter (already in SiteServiceRegistration.ObservedMeters). + ScadaBridgeTelemetry.RecordScriptCompileCacheEvictions(victims.Length); WarnEvicted(); } } diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbWiringTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbWiringTests.cs index 4171bed0..653a4c24 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbWiringTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbWiringTests.cs @@ -245,6 +245,17 @@ public class SiteLocalDbWiringTests : IDisposable Assert.Contains(LocalDbMetrics.MeterName, SiteServiceRegistration.ObservedMeters); } + /// Verifies that the ScadaBridge application meter (carrying the compile-cache eviction counter) is exported. + [Fact] + public void Site_Telemetry_Allowlists_TheScadaBridgeMeter() + { + // ScadaBridge#68: scadabridge.site.script_compile_cache.evictions lives on this meter; an + // unlisted meter is dropped silently (ZbTelemetryOptions.Meters is an allowlist). + Assert.Contains( + ZB.MOM.WW.ScadaBridge.Commons.Observability.ScadaBridgeTelemetry.MeterName, + SiteServiceRegistration.ObservedMeters); + } + /// Verifies that the Phase 2 tables are created in the consolidated LocalDb file. [Fact] public void Site_LocalDb_CreatesThePhase2Tables_InTheConsolidatedFile() diff --git a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/SiteScriptCompileCacheTests.cs b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/SiteScriptCompileCacheTests.cs index f28e568a..e5ffd14e 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/SiteScriptCompileCacheTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/SiteScriptCompileCacheTests.cs @@ -253,4 +253,32 @@ public class SiteScriptCompileCacheTests Assert.True(cache.Count <= 16); Assert.True(cache.Evictions > 0); } + + /// + /// ScadaBridge#68: evictions are exported as a Meter counter (not only the rate-limited Warning + /// log), on the ScadaBridge meter that the Host already allowlists for export. + /// + [Fact] + public void Eviction_IsExportedAsAMeterCounter() + { + long exported = 0; + using var listener = new System.Diagnostics.Metrics.MeterListener(); + listener.InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name == ZB.MOM.WW.ScadaBridge.Commons.Observability.ScadaBridgeTelemetry.MeterName + && instrument.Name == "scadabridge.site.script_compile_cache.evictions") + l.EnableMeasurementEvents(instrument); + }; + listener.SetMeasurementEventCallback((_, measurement, _, _) => Interlocked.Add(ref exported, measurement)); + listener.Start(); + + var cache = new SiteScriptCompileCache(16, logger: null); + for (var i = 0; i < 40; i++) + cache.GetOrAdd("return " + i + ";", typeof(ScriptGlobals), Ok); + + // The instrument is process-wide, so a parallel test's evictions may add to it: bound from below. + Assert.True(cache.Evictions > 0); + Assert.True(Interlocked.Read(ref exported) >= cache.Evictions, + $"exported {exported} but the cache evicted {cache.Evictions}"); + } }