feat(telemetry): export the site script compile-cache eviction counter as a Meter (ScadaBridge#68)
SiteScriptCompileCache now records each eviction sweep on scadabridge.site.script_compile_cache.evictions (ScadaBridgeTelemetry). It rides the existing ZB.MOM.WW.ScadaBridge meter, which is already in SiteServiceRegistration.ObservedMeters, so no new allowlist entry is needed; a Host test pins that the meter stays allowlisted.
This commit is contained in:
@@ -76,6 +76,17 @@ public static class ScadaBridgeTelemetry
|
|||||||
Meter.CreateCounter<long>("scadabridge.site.alarm_cache.buffer_dropped", unit: "1",
|
Meter.CreateCounter<long>("scadabridge.site.alarm_cache.buffer_dropped", unit: "1",
|
||||||
description: "Live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer (drop-oldest).");
|
description: "Live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer (drop-oldest).");
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Incremented by the number of entries the site script compile cache evicts when it crosses its
|
||||||
|
/// bound (ScadaBridge#68). Eviction does not unload the Roslyn assembly, so each recompile of an
|
||||||
|
/// evicted body loads another permanent copy: a sustained climb means compile-cache thrash and
|
||||||
|
/// monotonic working-set growth - raise <c>ScadaBridge:SiteRuntime:ScriptCompileCacheMaxEntries</c>.
|
||||||
|
/// Alert on this instead of the rate-limited Warning log.
|
||||||
|
/// </summary>
|
||||||
|
private static readonly Counter<long> _scriptCompileCacheEvictions =
|
||||||
|
Meter.CreateCounter<long>("scadabridge.site.script_compile_cache.evictions", unit: "1",
|
||||||
|
description: "Entries evicted from the site script compile cache at its bound (each recompile of an evicted body loads another permanent Roslyn assembly).");
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Incremented for each debug event evicted from a central debug session's bounded
|
/// Incremented for each debug event evicted from a central debug session's bounded
|
||||||
/// pre-snapshot buffer (drop-oldest). The Debug View is lossy-under-backpressure
|
/// pre-snapshot buffer (drop-oldest). The Debug View is lossy-under-backpressure
|
||||||
@@ -370,6 +381,10 @@ public static class ScadaBridgeTelemetry
|
|||||||
/// <param name="count">Number of deltas evicted.</param>
|
/// <param name="count">Number of deltas evicted.</param>
|
||||||
public static void RecordLiveAlarmBufferDrop(long count = 1) => _liveAlarmBufferDrops.Add(count);
|
public static void RecordLiveAlarmBufferDrop(long count = 1) => _liveAlarmBufferDrops.Add(count);
|
||||||
|
|
||||||
|
/// <summary>Records entries evicted from the site script compile cache.</summary>
|
||||||
|
/// <param name="count">Number of entries evicted by one sweep.</param>
|
||||||
|
public static void RecordScriptCompileCacheEvictions(long count) => _scriptCompileCacheEvictions.Add(count);
|
||||||
|
|
||||||
/// <summary>Records debug events evicted from a debug session's bounded pre-snapshot buffer.</summary>
|
/// <summary>Records debug events evicted from a debug session's bounded pre-snapshot buffer.</summary>
|
||||||
/// <param name="count">Number of events evicted.</param>
|
/// <param name="count">Number of events evicted.</param>
|
||||||
public static void RecordDebugPreSnapshotDrop(long count = 1) => _debugPreSnapshotDrops.Add(count);
|
public static void RecordDebugPreSnapshotDrop(long count = 1) => _debugPreSnapshotDrops.Add(count);
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
|
|||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||||
|
|
||||||
@@ -105,6 +106,8 @@ internal sealed class SiteScriptCompileCache
|
|||||||
/// <c>Script<object?></c>; the assembly Roslyn loaded for it stays in the default load context
|
/// <c>Script<object?></c>; the assembly Roslyn loaded for it stays in the default load context
|
||||||
/// for the process lifetime, and a later miss for the same body loads a second copy. A rising value
|
/// for the process lifetime, and a later miss for the same body loads a second copy. A rising value
|
||||||
/// therefore means monotonic working-set growth — raise <c>ScriptCompileCacheMaxEntries</c>.
|
/// therefore means monotonic working-set growth — raise <c>ScriptCompileCacheMaxEntries</c>.
|
||||||
|
/// Also exported as the <c>scadabridge.site.script_compile_cache.evictions</c> counter
|
||||||
|
/// (<see cref="ScadaBridgeTelemetry.RecordScriptCompileCacheEvictions"/>) for Prometheus alerting.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long Evictions => Interlocked.Read(ref _evictions);
|
public long Evictions => Interlocked.Read(ref _evictions);
|
||||||
|
|
||||||
@@ -183,6 +186,8 @@ internal sealed class SiteScriptCompileCache
|
|||||||
_cache.TryRemove(victim, out _);
|
_cache.TryRemove(victim, out _);
|
||||||
|
|
||||||
Interlocked.Add(ref _evictions, victims.Length);
|
Interlocked.Add(ref _evictions, victims.Length);
|
||||||
|
// Exported on the shared ScadaBridge meter (already in SiteServiceRegistration.ObservedMeters).
|
||||||
|
ScadaBridgeTelemetry.RecordScriptCompileCacheEvictions(victims.Length);
|
||||||
WarnEvicted();
|
WarnEvicted();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -245,6 +245,17 @@ public class SiteLocalDbWiringTests : IDisposable
|
|||||||
Assert.Contains(LocalDbMetrics.MeterName, SiteServiceRegistration.ObservedMeters);
|
Assert.Contains(LocalDbMetrics.MeterName, SiteServiceRegistration.ObservedMeters);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Verifies that the ScadaBridge application meter (carrying the compile-cache eviction counter) is exported.</summary>
|
||||||
|
[Fact]
|
||||||
|
public void Site_Telemetry_Allowlists_TheScadaBridgeMeter()
|
||||||
|
{
|
||||||
|
// ScadaBridge#68: scadabridge.site.script_compile_cache.evictions lives on this meter; an
|
||||||
|
// unlisted meter is dropped silently (ZbTelemetryOptions.Meters is an allowlist).
|
||||||
|
Assert.Contains(
|
||||||
|
ZB.MOM.WW.ScadaBridge.Commons.Observability.ScadaBridgeTelemetry.MeterName,
|
||||||
|
SiteServiceRegistration.ObservedMeters);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that the Phase 2 tables are created in the consolidated LocalDb file.</summary>
|
/// <summary>Verifies that the Phase 2 tables are created in the consolidated LocalDb file.</summary>
|
||||||
[Fact]
|
[Fact]
|
||||||
public void Site_LocalDb_CreatesThePhase2Tables_InTheConsolidatedFile()
|
public void Site_LocalDb_CreatesThePhase2Tables_InTheConsolidatedFile()
|
||||||
|
|||||||
@@ -253,4 +253,32 @@ public class SiteScriptCompileCacheTests
|
|||||||
Assert.True(cache.Count <= 16);
|
Assert.True(cache.Count <= 16);
|
||||||
Assert.True(cache.Evictions > 0);
|
Assert.True(cache.Evictions > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// ScadaBridge#68: evictions are exported as a Meter counter (not only the rate-limited Warning
|
||||||
|
/// log), on the ScadaBridge meter that the Host already allowlists for export.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Eviction_IsExportedAsAMeterCounter()
|
||||||
|
{
|
||||||
|
long exported = 0;
|
||||||
|
using var listener = new System.Diagnostics.Metrics.MeterListener();
|
||||||
|
listener.InstrumentPublished = (instrument, l) =>
|
||||||
|
{
|
||||||
|
if (instrument.Meter.Name == ZB.MOM.WW.ScadaBridge.Commons.Observability.ScadaBridgeTelemetry.MeterName
|
||||||
|
&& instrument.Name == "scadabridge.site.script_compile_cache.evictions")
|
||||||
|
l.EnableMeasurementEvents(instrument);
|
||||||
|
};
|
||||||
|
listener.SetMeasurementEventCallback<long>((_, measurement, _, _) => Interlocked.Add(ref exported, measurement));
|
||||||
|
listener.Start();
|
||||||
|
|
||||||
|
var cache = new SiteScriptCompileCache(16, logger: null);
|
||||||
|
for (var i = 0; i < 40; i++)
|
||||||
|
cache.GetOrAdd("return " + i + ";", typeof(ScriptGlobals), Ok);
|
||||||
|
|
||||||
|
// The instrument is process-wide, so a parallel test's evictions may add to it: bound from below.
|
||||||
|
Assert.True(cache.Evictions > 0);
|
||||||
|
Assert.True(Interlocked.Read(ref exported) >= cache.Evictions,
|
||||||
|
$"exported {exported} but the cache evicted {cache.Evictions}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user