fix(site-runtime): rate-limit shed Warning logs per script/alarm (review 2026-10-21 01/SR-P6)
This commit is contained in:
@@ -73,6 +73,7 @@ public class AlarmActor : ReceiveActor
|
||||
/// always counted; the event is rate-limited to one per alarm per minute.
|
||||
/// </summary>
|
||||
private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
|
||||
private int _shedSuppressed;
|
||||
|
||||
/// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
|
||||
private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
|
||||
@@ -864,12 +865,15 @@ public class AlarmActor : ReceiveActor
|
||||
var message = $"Alarm on-trigger script for '{_alarmName}' on instance '{_instanceName}': run shed — " +
|
||||
$"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
|
||||
|
||||
_logger.LogWarning("{Message}", message);
|
||||
|
||||
// Log + site event share one rate limit (review 2026-10-21 01/SR-P6); the counter above
|
||||
// counts every shed and the log line reports how many occurred since the last one.
|
||||
_shedSuppressed++;
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (now - _lastShedEventUtc >= ShedEventInterval)
|
||||
{
|
||||
_lastShedEventUtc = now;
|
||||
_logger.LogWarning("{Message} ({Shed} shed(s) since the last report)", message, _shedSuppressed);
|
||||
_shedSuppressed = 0;
|
||||
_ = _siteEventLogger?.LogEventAsync(
|
||||
"script", "Warning", _instanceName, $"AlarmActor:{_alarmName}", message);
|
||||
}
|
||||
|
||||
@@ -86,6 +86,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
|
||||
/// <c>site_events</c>.
|
||||
/// </summary>
|
||||
private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
|
||||
private int _shedSuppressed;
|
||||
|
||||
/// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
|
||||
private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
|
||||
@@ -678,14 +679,17 @@ public class ScriptActor : ReceiveActor, IWithTimers
|
||||
var message = $"Script '{_scriptName}' on instance '{_instanceName}': run shed — " +
|
||||
$"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
|
||||
|
||||
_logger.LogWarning("{Message}", message);
|
||||
|
||||
// Rate-limited to one event per script per minute: the counter above still counts
|
||||
// every shed, but a hot trigger against a saturated cap must not flood site_events.
|
||||
// Rate-limited to one log line + one event per script per minute: the counter above
|
||||
// still counts every shed, but a hot trigger against a saturated cap must not flood
|
||||
// site_events or the node log (review 2026-10-21 01/SR-P6). The first shed logs at once;
|
||||
// later lines carry the number of sheds suppressed since the previous one.
|
||||
_shedSuppressed++;
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (now - _lastShedEventUtc >= ShedEventInterval)
|
||||
{
|
||||
_lastShedEventUtc = now;
|
||||
_logger.LogWarning("{Message} ({Shed} shed(s) since the last report)", message, _shedSuppressed);
|
||||
_shedSuppressed = 0;
|
||||
_ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
|
||||
"script", "Warning", _instanceName, $"ScriptActor:{_scriptName}", message);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user