lmxopcua/src/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Host/Stability/MemoryWatchdog.cs

using System;
using System.Collections.Generic;

namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Host.Stability;

/// <summary>
///     Galaxy-specific RSS watchdog per <c>driver-stability.md §"Memory Watchdog Thresholds"</c>.
///     Baseline-relative + absolute caps. Sustained-slope detection uses a rolling 30-min window.
///     Pluggable RSS source keeps it unit-testable.
/// </summary>
public sealed class MemoryWatchdog
{
    /// <summary>Absolute hard ceiling — process is force-killed above this.</summary>
    public long HardCeilingBytes { get; init; } = 1_500L * 1024 * 1024;

    /// <summary>Sustained slope (bytes/min) above which soft recycle is scheduled.</summary>
    public long SustainedSlopeBytesPerMinute { get; init; } = 5L * 1024 * 1024;

    public TimeSpan SlopeWindow { get; init; } = TimeSpan.FromMinutes(30);

    private readonly long _baselineBytes;
    private readonly Queue<RssSample> _samples = new();

    public MemoryWatchdog(long baselineBytes)
    {
        _baselineBytes = baselineBytes;
    }

    /// <summary>Called every 30s with the current RSS. Returns the action the supervisor should take.</summary>
    public WatchdogAction Sample(long rssBytes, DateTime utcNow)
    {
        _samples.Enqueue(new RssSample(utcNow, rssBytes));
        while (_samples.Count > 0 && utcNow - _samples.Peek().TimestampUtc > SlopeWindow)
            _samples.Dequeue();

        if (rssBytes >= HardCeilingBytes)
            return WatchdogAction.HardKill;

        var softThreshold = Math.Max(_baselineBytes * 2, _baselineBytes + 200L * 1024 * 1024);
        var warnThreshold = Math.Max((long)(_baselineBytes * 1.5), _baselineBytes + 200L * 1024 * 1024);

        if (rssBytes >= softThreshold) return WatchdogAction.SoftRecycle;
        if (rssBytes >= warnThreshold) return WatchdogAction.Warn;

        if (_samples.Count >= 2)
        {
            var oldest = _samples.Peek();
            var span = (utcNow - oldest.TimestampUtc).TotalMinutes;
            if (span >= SlopeWindow.TotalMinutes * 0.9) // need ~full window to trust the slope
            {
                var delta = rssBytes - oldest.RssBytes;
                var bytesPerMin = delta / span;
                if (bytesPerMin >= SustainedSlopeBytesPerMinute)
                    return WatchdogAction.SoftRecycle;
            }
        }

        return WatchdogAction.None;
    }

    private readonly record struct RssSample(DateTime TimestampUtc, long RssBytes);
}

public enum WatchdogAction { None, Warn, SoftRecycle, HardKill }