fix(site-runtime): rate-limit shed Warning logs per script/alarm (review 2026-10-21 01/SR-P6)
This commit is contained in:
@@ -86,6 +86,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
|
||||
/// <c>site_events</c>.
|
||||
/// </summary>
|
||||
private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
|
||||
private int _shedSuppressed;
|
||||
|
||||
/// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
|
||||
private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
|
||||
@@ -678,14 +679,17 @@ public class ScriptActor : ReceiveActor, IWithTimers
|
||||
var message = $"Script '{_scriptName}' on instance '{_instanceName}': run shed — " +
|
||||
$"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
|
||||
|
||||
_logger.LogWarning("{Message}", message);
|
||||
|
||||
// Rate-limited to one event per script per minute: the counter above still counts
|
||||
// every shed, but a hot trigger against a saturated cap must not flood site_events.
|
||||
// Rate-limited to one log line + one event per script per minute: the counter above
|
||||
// still counts every shed, but a hot trigger against a saturated cap must not flood
|
||||
// site_events or the node log (review 2026-10-21 01/SR-P6). The first shed logs at once;
|
||||
// later lines carry the number of sheds suppressed since the previous one.
|
||||
_shedSuppressed++;
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (now - _lastShedEventUtc >= ShedEventInterval)
|
||||
{
|
||||
_lastShedEventUtc = now;
|
||||
_logger.LogWarning("{Message} ({Shed} shed(s) since the last report)", message, _shedSuppressed);
|
||||
_shedSuppressed = 0;
|
||||
_ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
|
||||
"script", "Warning", _instanceName, $"ScriptActor:{_scriptName}", message);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user