fix(site-runtime): rate-limit shed Warning logs per script/alarm (review 2026-10-21 01/SR-P6)

This commit is contained in:
Joseph Doherty
2026-10-22 08:29:50 -04:00
parent 7541a38762
commit 4ff26a0614
3 changed files with 58 additions and 6 deletions
@@ -73,6 +73,7 @@ public class AlarmActor : ReceiveActor
/// always counted; the event is rate-limited to one per alarm per minute.
/// </summary>
private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
private int _shedSuppressed;
/// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
@@ -864,12 +865,15 @@ public class AlarmActor : ReceiveActor
var message = $"Alarm on-trigger script for '{_alarmName}' on instance '{_instanceName}': run shed — " +
$"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
_logger.LogWarning("{Message}", message);
// Log + site event share one rate limit (review 2026-10-21 01/SR-P6); the counter above
// counts every shed and the log line reports how many occurred since the last one.
_shedSuppressed++;
var now = DateTimeOffset.UtcNow;
if (now - _lastShedEventUtc >= ShedEventInterval)
{
_lastShedEventUtc = now;
_logger.LogWarning("{Message} ({Shed} shed(s) since the last report)", message, _shedSuppressed);
_shedSuppressed = 0;
_ = _siteEventLogger?.LogEventAsync(
"script", "Warning", _instanceName, $"AlarmActor:{_alarmName}", message);
}
@@ -86,6 +86,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// <c>site_events</c>.
/// </summary>
private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
private int _shedSuppressed;
/// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
@@ -678,14 +679,17 @@ public class ScriptActor : ReceiveActor, IWithTimers
var message = $"Script '{_scriptName}' on instance '{_instanceName}': run shed — " +
$"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
_logger.LogWarning("{Message}", message);
// Rate-limited to one event per script per minute: the counter above still counts
// every shed, but a hot trigger against a saturated cap must not flood site_events.
// Rate-limited to one log line + one event per script per minute: the counter above
// still counts every shed, but a hot trigger against a saturated cap must not flood
// site_events or the node log (review 2026-10-21 01/SR-P6). The first shed logs at once;
// later lines carry the number of sheds suppressed since the previous one.
_shedSuppressed++;
var now = DateTimeOffset.UtcNow;
if (now - _lastShedEventUtc >= ShedEventInterval)
{
_lastShedEventUtc = now;
_logger.LogWarning("{Message} ({Shed} shed(s) since the last report)", message, _shedSuppressed);
_shedSuppressed = 0;
_ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
"script", "Warning", _instanceName, $"ScriptActor:{_scriptName}", message);
}