fix(host): rate-limit dead-letter warnings (10/min + suppression summary); metric counting unchanged

This commit is contained in:
Joseph Doherty
2026-07-08 16:52:25 -04:00
parent 3ce21734b8
commit dea69842d5
3 changed files with 89 additions and 6 deletions
@@ -8,12 +8,31 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Actors;
/// <summary>
/// Subscribes to Akka.NET dead letter events, logs them, and tracks count
/// for health monitoring integration.
///
/// Warning logging is rate-limited to <see cref="MaxWarningsPerWindow"/> per
/// <see cref="WarningWindow"/>: a failover/undeploy storm can produce thousands
/// of dead letters, and one Warning per letter is an unbounded log flood
/// (review 01 [Low]). Once the per-window cap is hit, per-letter warnings are
/// suppressed and a single summary of the suppressed count is logged when the
/// window rolls over. The health-metric count is <b>never</b> throttled — every
/// dead letter is counted.
/// </summary>
public class DeadLetterMonitorActor : ReceiveActor
public class DeadLetterMonitorActor : ReceiveActor, IWithTimers
{
/// <summary>Maximum per-letter Warning logs emitted within one <see cref="WarningWindow"/>.</summary>
public const int MaxWarningsPerWindow = 10;
/// <summary>Rolling window over which <see cref="MaxWarningsPerWindow"/> applies.</summary>
internal static readonly TimeSpan WarningWindow = TimeSpan.FromMinutes(1);
private long _deadLetterCount;
private int _warningsThisWindow;
private long _suppressedThisWindow;
private readonly ISiteHealthCollector? _healthCollector;
/// <summary>Akka timer scheduler (assigned by the runtime via <see cref="IWithTimers"/>).</summary>
public ITimerScheduler Timers { get; set; } = null!;
/// <summary>
/// Initializes the actor and registers dead-letter message handlers.
/// </summary>
@@ -25,13 +44,39 @@ public class DeadLetterMonitorActor : ReceiveActor
Receive<DeadLetter>(dl =>
{
// Metric counting is never throttled.
_deadLetterCount++;
_healthCollector?.IncrementDeadLetter();
logger.LogWarning(
"Dead letter: {MessageType} from {Sender} to {Recipient}",
dl.Message.GetType().Name,
dl.Sender,
dl.Recipient);
if (_warningsThisWindow < MaxWarningsPerWindow)
{
_warningsThisWindow++;
logger.LogWarning(
"Dead letter: {MessageType} from {Sender} to {Recipient}",
dl.Message.GetType().Name,
dl.Sender,
dl.Recipient);
if (_warningsThisWindow == MaxWarningsPerWindow)
logger.LogWarning(
"Dead-letter warning limit ({Limit}) reached — suppressing per-letter warnings for {Window}",
MaxWarningsPerWindow, WarningWindow);
}
else
{
_suppressedThisWindow++;
}
});
Receive<WindowTick>(_ =>
{
if (_suppressedThisWindow > 0)
logger.LogWarning(
"Suppressed {Count} dead-letter warnings in the last {Window}",
_suppressedThisWindow, WarningWindow);
_warningsThisWindow = 0;
_suppressedThisWindow = 0;
});
Receive<GetDeadLetterCount>(_ => Sender.Tell(new DeadLetterCountResponse(_deadLetterCount)));
@@ -41,6 +86,7 @@ public class DeadLetterMonitorActor : ReceiveActor
protected override void PreStart()
{
Context.System.EventStream.Subscribe(Self, typeof(DeadLetter));
Timers.StartPeriodicTimer("dl-window", WindowTick.Instance, WarningWindow);
}
/// <inheritdoc />
@@ -48,6 +94,13 @@ public class DeadLetterMonitorActor : ReceiveActor
{
Context.System.EventStream.Unsubscribe(Self, typeof(DeadLetter));
}
/// <summary>Periodic self-message that rolls the warning-rate-limit window.</summary>
private sealed class WindowTick
{
public static readonly WindowTick Instance = new();
private WindowTick() { }
}
}
/// <summary>