Merge pull request (#79) - Phase 6.1 Stream B - Tier A/B/C stability (registry + MemoryTracking + MemoryRecycle + Scheduler + WedgeDetector)

src/ZB.MOM.WW.OtOpcUa.Core.Abstractions/DriverTypeRegistry.cs

@@ -69,12 +69,20 @@ public sealed class DriverTypeRegistry
 /// <param name="DriverConfigJsonSchema">JSON Schema (Draft 2020-12) the driver's <c>DriverConfig</c> column must validate against.</param>
 /// <param name="DeviceConfigJsonSchema">JSON Schema for <c>DeviceConfig</c> (multi-device drivers); null if the driver has no device layer.</param>
 /// <param name="TagConfigJsonSchema">JSON Schema for <c>TagConfig</c>; required for every driver since every driver has tags.</param>
+/// <param name="Tier">
+///     Stability tier per <c>docs/v2/driver-stability.md</c> §2-4 and <c>docs/v2/plan.md</c>
+///     decisions #63-74. Drives the shared resilience pipeline defaults
+///     (<see cref="Tier"/> × capability → <c>CapabilityPolicy</c>), the <c>MemoryTracking</c>
+///     hybrid-formula constants, and whether process-level <c>MemoryRecycle</c> / scheduled-
+///     recycle protections apply (Tier C only). Every registered driver type must declare one.
+/// </param>
 public sealed record DriverTypeMetadata(
     string TypeName,
     NamespaceKindCompatibility AllowedNamespaceKinds,
     string DriverConfigJsonSchema,
     string? DeviceConfigJsonSchema,
-    string TagConfigJsonSchema);
+    string TagConfigJsonSchema,
+    DriverTier Tier);
 
 /// <summary>Bitmask of namespace kinds a driver type may populate. Per decision #111.</summary>
 [Flags]

src/ZB.MOM.WW.OtOpcUa.Core.Abstractions/IDriverSupervisor.cs

@@ -0,0 +1,26 @@
+namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+/// <summary>
+///     Process-level supervisor contract a Tier C driver's out-of-process topology provides
+///     (e.g. <c>Driver.Galaxy.Proxy/Supervisor/</c>). Concerns: restart the Host process when a
+///     hard fault is detected (memory breach, wedge, scheduled recycle window).
+/// </summary>
+/// <remarks>
+///     Per <c>docs/v2/plan.md</c> decisions #68, #73-74, and #145. Tier A/B drivers do NOT have
+///     a supervisor because they run in-process — recycling would kill every OPC UA session and
+///     every co-hosted driver. The Core.Stability layer only invokes this interface for Tier C
+///     instances after asserting the tier via <see cref="DriverTypeMetadata.Tier"/>.
+/// </remarks>
+public interface IDriverSupervisor
+{
+    /// <summary>Driver instance this supervisor governs.</summary>
+    string DriverInstanceId { get; }
+
+    /// <summary>
+    ///     Request the supervisor to recycle (terminate + restart) the Host process. Implementations
+    ///     are expected to be idempotent under repeat calls during an in-flight recycle.
+    /// </summary>
+    /// <param name="reason">Human-readable reason — flows into the supervisor's logs.</param>
+    /// <param name="cancellationToken">Cancels the recycle request; an in-flight restart is not interrupted.</param>
+    Task RecycleAsync(string reason, CancellationToken cancellationToken);
+}

src/ZB.MOM.WW.OtOpcUa.Core/Stability/MemoryRecycle.cs

@@ -0,0 +1,65 @@
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Stability;
+
+/// <summary>
+///     Tier C only process-recycle companion to <see cref="MemoryTracking"/>. On a
+///     <see cref="MemoryTrackingAction.HardBreach"/> signal, invokes the supplied
+///     <see cref="IDriverSupervisor"/> to restart the out-of-process Host.
+/// </summary>
+/// <remarks>
+///     Per <c>docs/v2/plan.md</c> decisions #74 and #145. Tier A/B hard-breach on an in-process
+///     driver would kill every OPC UA session and every co-hosted driver, so for Tier A/B this
+///     class logs a <b>promotion-to-Tier-C recommendation</b> and does NOT invoke any supervisor.
+///     A future tier-migration workflow acts on the recommendation.
+/// </remarks>
+public sealed class MemoryRecycle
+{
+    private readonly DriverTier _tier;
+    private readonly IDriverSupervisor? _supervisor;
+    private readonly ILogger<MemoryRecycle> _logger;
+
+    public MemoryRecycle(DriverTier tier, IDriverSupervisor? supervisor, ILogger<MemoryRecycle> logger)
+    {
+        _tier = tier;
+        _supervisor = supervisor;
+        _logger = logger;
+    }
+
+    /// <summary>
+    ///     Handle a <see cref="MemoryTracking"/> classification for the driver. For Tier C with a
+    ///     wired supervisor, <c>HardBreach</c> triggers <see cref="IDriverSupervisor.RecycleAsync"/>.
+    ///     All other combinations are no-ops with respect to process state (soft breaches + Tier A/B
+    ///     hard breaches just log).
+    /// </summary>
+    /// <returns>True when a recycle was requested; false otherwise.</returns>
+    public async Task<bool> HandleAsync(MemoryTrackingAction action, long footprintBytes, CancellationToken cancellationToken)
+    {
+        switch (action)
+        {
+            case MemoryTrackingAction.SoftBreach:
+                _logger.LogWarning(
+                    "Memory soft-breach on driver {DriverId}: footprint={Footprint:N0} bytes, tier={Tier}. Surfaced to Admin; no action.",
+                    _supervisor?.DriverInstanceId ?? "(unknown)", footprintBytes, _tier);
+                return false;
+
+            case MemoryTrackingAction.HardBreach when _tier == DriverTier.C && _supervisor is not null:
+                _logger.LogError(
+                    "Memory hard-breach on Tier C driver {DriverId}: footprint={Footprint:N0} bytes. Requesting supervisor recycle.",
+                    _supervisor.DriverInstanceId, footprintBytes);
+                await _supervisor.RecycleAsync($"Memory hard-breach: {footprintBytes} bytes", cancellationToken).ConfigureAwait(false);
+                return true;
+
+            case MemoryTrackingAction.HardBreach:
+                _logger.LogError(
+                    "Memory hard-breach on Tier {Tier} in-process driver {DriverId}: footprint={Footprint:N0} bytes. " +
+                    "Recommending promotion to Tier C; NOT auto-killing (decisions #74, #145).",
+                    _tier, _supervisor?.DriverInstanceId ?? "(unknown)", footprintBytes);
+                return false;
+
+            default:
+                return false;
+        }
+    }
+}

src/ZB.MOM.WW.OtOpcUa.Core/Stability/MemoryTracking.cs

@@ -0,0 +1,136 @@
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Stability;
+
+/// <summary>
+///     Tier-agnostic memory-footprint tracker. Captures the post-initialize <b>baseline</b>
+///     from the first samples after <c>IDriver.InitializeAsync</c>, then classifies each
+///     subsequent sample against a hybrid soft/hard threshold per
+///     <c>docs/v2/plan.md</c> decision #146 — <c>soft = max(multiplier × baseline, baseline + floor)</c>,
+///     <c>hard = 2 × soft</c>.
+/// </summary>
+/// <remarks>
+///     <para>Per decision #145, this tracker <b>never kills a process</b>. Soft and hard breaches
+///     log + surface to the Admin UI via <c>DriverInstanceResilienceStatus</c>. The matching
+///     process-level recycle protection lives in a separate <c>MemoryRecycle</c> that activates
+///     for Tier C drivers only (where the driver runs out-of-process behind a supervisor that
+///     can safely restart it without tearing down the OPC UA session or co-hosted in-proc
+///     drivers).</para>
+///
+///     <para>Baseline capture: the tracker starts in <see cref="TrackingPhase.WarmingUp"/> for
+///     <see cref="BaselineWindow"/> (default 5 min). During that window samples are collected;
+///     the baseline is computed as the median once the window elapses. Before that point every
+///     classification returns <see cref="MemoryTrackingAction.Warming"/>.</para>
+/// </remarks>
+public sealed class MemoryTracking
+{
+    private readonly DriverTier _tier;
+    private readonly TimeSpan _baselineWindow;
+    private readonly List<long> _warmupSamples = [];
+    private long _baselineBytes;
+    private TrackingPhase _phase = TrackingPhase.WarmingUp;
+    private DateTime? _warmupStartUtc;
+
+    /// <summary>Tier-default multiplier/floor constants per decision #146.</summary>
+    public static (int Multiplier, long FloorBytes) GetTierConstants(DriverTier tier) => tier switch
+    {
+        DriverTier.A => (Multiplier: 3, FloorBytes: 50L * 1024 * 1024),
+        DriverTier.B => (Multiplier: 3, FloorBytes: 100L * 1024 * 1024),
+        DriverTier.C => (Multiplier: 2, FloorBytes: 500L * 1024 * 1024),
+        _ => throw new ArgumentOutOfRangeException(nameof(tier), tier, $"No memory-tracking constants defined for tier {tier}."),
+    };
+
+    /// <summary>Window over which post-init samples are collected to compute the baseline.</summary>
+    public TimeSpan BaselineWindow => _baselineWindow;
+
+    /// <summary>Current phase: <see cref="TrackingPhase.WarmingUp"/> or <see cref="TrackingPhase.Steady"/>.</summary>
+    public TrackingPhase Phase => _phase;
+
+    /// <summary>Captured baseline; 0 until warmup completes.</summary>
+    public long BaselineBytes => _baselineBytes;
+
+    /// <summary>Effective soft threshold (zero while warming up).</summary>
+    public long SoftThresholdBytes => _baselineBytes == 0 ? 0 : ComputeSoft(_tier, _baselineBytes);
+
+    /// <summary>Effective hard threshold = 2 × soft (zero while warming up).</summary>
+    public long HardThresholdBytes => _baselineBytes == 0 ? 0 : ComputeSoft(_tier, _baselineBytes) * 2;
+
+    public MemoryTracking(DriverTier tier, TimeSpan? baselineWindow = null)
+    {
+        _tier = tier;
+        _baselineWindow = baselineWindow ?? TimeSpan.FromMinutes(5);
+    }
+
+    /// <summary>
+    ///     Submit a memory-footprint sample. Returns the action the caller should surface.
+    ///     During warmup, always returns <see cref="MemoryTrackingAction.Warming"/> and accumulates
+    ///     samples; once the window elapses the first steady-phase sample triggers baseline capture
+    ///     (median of warmup samples).
+    /// </summary>
+    public MemoryTrackingAction Sample(long footprintBytes, DateTime utcNow)
+    {
+        if (_phase == TrackingPhase.WarmingUp)
+        {
+            _warmupStartUtc ??= utcNow;
+            _warmupSamples.Add(footprintBytes);
+            if (utcNow - _warmupStartUtc.Value >= _baselineWindow && _warmupSamples.Count > 0)
+            {
+                _baselineBytes = ComputeMedian(_warmupSamples);
+                _phase = TrackingPhase.Steady;
+            }
+            else
+            {
+                return MemoryTrackingAction.Warming;
+            }
+        }
+
+        if (footprintBytes >= HardThresholdBytes) return MemoryTrackingAction.HardBreach;
+        if (footprintBytes >= SoftThresholdBytes) return MemoryTrackingAction.SoftBreach;
+        return MemoryTrackingAction.None;
+    }
+
+    private static long ComputeSoft(DriverTier tier, long baseline)
+    {
+        var (multiplier, floor) = GetTierConstants(tier);
+        return Math.Max(multiplier * baseline, baseline + floor);
+    }
+
+    private static long ComputeMedian(List<long> samples)
+    {
+        var sorted = samples.Order().ToArray();
+        var mid = sorted.Length / 2;
+        return sorted.Length % 2 == 1
+            ? sorted[mid]
+            : (sorted[mid - 1] + sorted[mid]) / 2;
+    }
+}
+
+/// <summary>Phase of a <see cref="MemoryTracking"/> lifecycle.</summary>
+public enum TrackingPhase
+{
+    /// <summary>Collecting post-init samples; baseline not yet computed.</summary>
+    WarmingUp,
+
+    /// <summary>Baseline captured; every sample classified against soft/hard thresholds.</summary>
+    Steady,
+}
+
+/// <summary>Classification the tracker returns per sample.</summary>
+public enum MemoryTrackingAction
+{
+    /// <summary>Baseline not yet captured; sample collected, no threshold check.</summary>
+    Warming,
+
+    /// <summary>Below soft threshold.</summary>
+    None,
+
+    /// <summary>Between soft and hard thresholds — log + surface, no action.</summary>
+    SoftBreach,
+
+    /// <summary>
+    ///     ≥ hard threshold. Log + surface + (Tier C only, via <c>MemoryRecycle</c>) request
+    ///     process recycle via the driver supervisor. Tier A/B breach never invokes any
+    ///     kill path per decisions #145 and #74.
+    /// </summary>
+    HardBreach,
+}

src/ZB.MOM.WW.OtOpcUa.Core/Stability/ScheduledRecycleScheduler.cs

@@ -0,0 +1,86 @@
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Stability;
+
+/// <summary>
+///     Tier C opt-in periodic-recycle driver per <c>docs/v2/plan.md</c> decision #67.
+///     A tick method advanced by the caller (fed by a background timer in prod; by test clock
+///     in unit tests) decides whether the configured interval has elapsed and, if so, drives the
+///     supplied <see cref="IDriverSupervisor"/> to recycle the Host.
+/// </summary>
+/// <remarks>
+///     Tier A/B drivers MUST NOT use this class — scheduled recycle for in-process drivers would
+///     kill every OPC UA session and every co-hosted driver. The ctor throws when constructed
+///     with any tier other than C to make the misuse structurally impossible.
+///
+///     <para>Keeps no background thread of its own — callers invoke <see cref="TickAsync"/> on
+///     their ambient scheduler tick (Phase 6.1 Stream C's health-endpoint host runs one). That
+///     decouples the unit under test from wall-clock time and thread-pool scheduling.</para>
+/// </remarks>
+public sealed class ScheduledRecycleScheduler
+{
+    private readonly TimeSpan _recycleInterval;
+    private readonly IDriverSupervisor _supervisor;
+    private readonly ILogger<ScheduledRecycleScheduler> _logger;
+    private DateTime _nextRecycleUtc;
+
+    /// <summary>
+    ///     Construct the scheduler for a Tier C driver. Throws if <paramref name="tier"/> isn't C.
+    /// </summary>
+    /// <param name="tier">Driver tier; must be <see cref="DriverTier.C"/>.</param>
+    /// <param name="recycleInterval">Interval between recycles (e.g. 7 days).</param>
+    /// <param name="startUtc">Anchor time; next recycle fires at <paramref name="startUtc"/> + <paramref name="recycleInterval"/>.</param>
+    /// <param name="supervisor">Supervisor that performs the actual recycle.</param>
+    /// <param name="logger">Diagnostic sink.</param>
+    public ScheduledRecycleScheduler(
+        DriverTier tier,
+        TimeSpan recycleInterval,
+        DateTime startUtc,
+        IDriverSupervisor supervisor,
+        ILogger<ScheduledRecycleScheduler> logger)
+    {
+        if (tier != DriverTier.C)
+            throw new ArgumentException(
+                $"ScheduledRecycleScheduler is Tier C only (got {tier}). " +
+                "In-process drivers must not use scheduled recycle; see decisions #74 and #145.",
+                nameof(tier));
+
+        if (recycleInterval <= TimeSpan.Zero)
+            throw new ArgumentException("RecycleInterval must be positive.", nameof(recycleInterval));
+
+        _recycleInterval = recycleInterval;
+        _supervisor = supervisor;
+        _logger = logger;
+        _nextRecycleUtc = startUtc + recycleInterval;
+    }
+
+    /// <summary>Next scheduled recycle UTC. Advances by <see cref="RecycleInterval"/> on each fire.</summary>
+    public DateTime NextRecycleUtc => _nextRecycleUtc;
+
+    /// <summary>Recycle interval this scheduler was constructed with.</summary>
+    public TimeSpan RecycleInterval => _recycleInterval;
+
+    /// <summary>
+    ///     Tick the scheduler forward. If <paramref name="utcNow"/> is past
+    ///     <see cref="NextRecycleUtc"/>, requests a recycle from the supervisor and advances
+    ///     <see cref="NextRecycleUtc"/> by exactly one interval. Returns true when a recycle fired.
+    /// </summary>
+    public async Task<bool> TickAsync(DateTime utcNow, CancellationToken cancellationToken)
+    {
+        if (utcNow < _nextRecycleUtc)
+            return false;
+
+        _logger.LogInformation(
+            "Scheduled recycle due for Tier C driver {DriverId} at {Now:o}; advancing next to {Next:o}.",
+            _supervisor.DriverInstanceId, utcNow, _nextRecycleUtc + _recycleInterval);
+
+        await _supervisor.RecycleAsync("Scheduled periodic recycle", cancellationToken).ConfigureAwait(false);
+        _nextRecycleUtc += _recycleInterval;
+        return true;
+    }
+
+    /// <summary>Request an immediate recycle outside the schedule (e.g. MemoryRecycle hard-breach escalation).</summary>
+    public Task RequestRecycleNowAsync(string reason, CancellationToken cancellationToken) =>
+        _supervisor.RecycleAsync(reason, cancellationToken);
+}

src/ZB.MOM.WW.OtOpcUa.Core/Stability/WedgeDetector.cs

@@ -0,0 +1,81 @@
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Stability;
+
+/// <summary>
+///     Demand-aware driver-wedge detector per <c>docs/v2/plan.md</c> decision #147.
+///     Flips a driver to <see cref="WedgeVerdict.Faulted"/> only when BOTH of the following hold:
+///     (a) there is pending work outstanding, AND (b) no progress has been observed for longer
+///     than <see cref="Threshold"/>. Idle drivers, write-only burst drivers, and subscription-only
+///     drivers whose signals don't arrive regularly all stay Healthy.
+/// </summary>
+/// <remarks>
+///     <para>Pending work signal is supplied by the caller via <see cref="DemandSignal"/>:
+///     non-zero Polly bulkhead depth, ≥1 active MonitoredItem, or ≥1 queued historian read
+///     each qualifies. The detector itself is state-light: all it remembers is the last
+///     <c>LastProgressUtc</c> it saw and the last wedge verdict. No history buffer.</para>
+///
+///     <para>Default threshold per plan: <c>5 × PublishingInterval</c>, with a minimum of 60 s.
+///     Concrete values are driver-agnostic and configured per-instance by the caller.</para>
+/// </remarks>
+public sealed class WedgeDetector
+{
+    /// <summary>Wedge-detection threshold; pass &lt; 60 s and the detector clamps to 60 s.</summary>
+    public TimeSpan Threshold { get; }
+
+    /// <summary>Whether the driver reported itself <see cref="DriverState.Healthy"/> at construction.</summary>
+    public WedgeDetector(TimeSpan threshold)
+    {
+        Threshold = threshold < TimeSpan.FromSeconds(60) ? TimeSpan.FromSeconds(60) : threshold;
+    }
+
+    /// <summary>
+    ///     Classify the current state against the demand signal. Does not retain state across
+    ///     calls — each call is self-contained; the caller owns the <c>LastProgressUtc</c> clock.
+    /// </summary>
+    public WedgeVerdict Classify(DriverState state, DemandSignal demand, DateTime utcNow)
+    {
+        if (state != DriverState.Healthy)
+            return WedgeVerdict.NotApplicable;
+
+        if (!demand.HasPendingWork)
+            return WedgeVerdict.Idle;
+
+        var sinceProgress = utcNow - demand.LastProgressUtc;
+        return sinceProgress > Threshold ? WedgeVerdict.Faulted : WedgeVerdict.Healthy;
+    }
+}
+
+/// <summary>
+///     Caller-supplied demand snapshot. All three counters are OR'd — any non-zero means work
+///     is outstanding, which is the trigger for checking the <see cref="LastProgressUtc"/> clock.
+/// </summary>
+/// <param name="BulkheadDepth">Polly bulkhead depth (in-flight capability calls).</param>
+/// <param name="ActiveMonitoredItems">Number of live OPC UA MonitoredItems bound to this driver.</param>
+/// <param name="QueuedHistoryReads">Pending historian-read requests the driver owes the server.</param>
+/// <param name="LastProgressUtc">Last time the driver reported a successful unit of work (read, subscribe-ack, publish).</param>
+public readonly record struct DemandSignal(
+    int BulkheadDepth,
+    int ActiveMonitoredItems,
+    int QueuedHistoryReads,
+    DateTime LastProgressUtc)
+{
+    /// <summary>True when any of the three counters is &gt; 0.</summary>
+    public bool HasPendingWork => BulkheadDepth > 0 || ActiveMonitoredItems > 0 || QueuedHistoryReads > 0;
+}
+
+/// <summary>Outcome of a single <see cref="WedgeDetector.Classify"/> call.</summary>
+public enum WedgeVerdict
+{
+    /// <summary>Driver wasn't Healthy to begin with — wedge detection doesn't apply.</summary>
+    NotApplicable,
+
+    /// <summary>Driver claims Healthy + no pending work → stays Healthy.</summary>
+    Idle,
+
+    /// <summary>Driver claims Healthy + has pending work + has made progress within the threshold → stays Healthy.</summary>
+    Healthy,
+
+    /// <summary>Driver claims Healthy + has pending work + has NOT made progress within the threshold → wedged.</summary>
+    Faulted,
+}

tests/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/DriverTypeRegistryTests.cs

@@ -7,11 +7,13 @@ public sealed class DriverTypeRegistryTests
 {
     private static DriverTypeMetadata SampleMetadata(
         string typeName = "Modbus",
-        NamespaceKindCompatibility allowed = NamespaceKindCompatibility.Equipment) =>
+        NamespaceKindCompatibility allowed = NamespaceKindCompatibility.Equipment,
+        DriverTier tier = DriverTier.B) =>
         new(typeName, allowed,
             DriverConfigJsonSchema: "{\"type\": \"object\"}",
             DeviceConfigJsonSchema: "{\"type\": \"object\"}",
-            TagConfigJsonSchema: "{\"type\": \"object\"}");
+            TagConfigJsonSchema: "{\"type\": \"object\"}",
+            Tier: tier);
 
     [Fact]
     public void Register_ThenGet_RoundTrips()
@@ -24,6 +26,20 @@ public sealed class DriverTypeRegistryTests
         registry.Get("Modbus").ShouldBe(metadata);
     }
 
+    [Theory]
+    [InlineData(DriverTier.A)]
+    [InlineData(DriverTier.B)]
+    [InlineData(DriverTier.C)]
+    public void Register_Requires_NonNullTier(DriverTier tier)
+    {
+        var registry = new DriverTypeRegistry();
+        var metadata = SampleMetadata(typeName: $"Driver-{tier}", tier: tier);
+
+        registry.Register(metadata);
+
+        registry.Get(metadata.TypeName).Tier.ShouldBe(tier);
+    }
+
     [Fact]
     public void Get_IsCaseInsensitive()
     {

tests/ZB.MOM.WW.OtOpcUa.Core.Tests/Stability/MemoryRecycleTests.cs

@@ -0,0 +1,91 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Stability;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Stability;
+
+[Trait("Category", "Unit")]
+public sealed class MemoryRecycleTests
+{
+    [Fact]
+    public async Task TierC_HardBreach_RequestsSupervisorRecycle()
+    {
+        var supervisor = new FakeSupervisor();
+        var recycle = new MemoryRecycle(DriverTier.C, supervisor, NullLogger<MemoryRecycle>.Instance);
+
+        var requested = await recycle.HandleAsync(MemoryTrackingAction.HardBreach, 2_000_000_000, CancellationToken.None);
+
+        requested.ShouldBeTrue();
+        supervisor.RecycleCount.ShouldBe(1);
+        supervisor.LastReason.ShouldContain("hard-breach");
+    }
+
+    [Theory]
+    [InlineData(DriverTier.A)]
+    [InlineData(DriverTier.B)]
+    public async Task InProcessTier_HardBreach_NeverRequestsRecycle(DriverTier tier)
+    {
+        var supervisor = new FakeSupervisor();
+        var recycle = new MemoryRecycle(tier, supervisor, NullLogger<MemoryRecycle>.Instance);
+
+        var requested = await recycle.HandleAsync(MemoryTrackingAction.HardBreach, 2_000_000_000, CancellationToken.None);
+
+        requested.ShouldBeFalse("Tier A/B hard-breach logs a promotion recommendation only (decisions #74, #145)");
+        supervisor.RecycleCount.ShouldBe(0);
+    }
+
+    [Fact]
+    public async Task TierC_WithoutSupervisor_HardBreach_NoOp()
+    {
+        var recycle = new MemoryRecycle(DriverTier.C, supervisor: null, NullLogger<MemoryRecycle>.Instance);
+
+        var requested = await recycle.HandleAsync(MemoryTrackingAction.HardBreach, 2_000_000_000, CancellationToken.None);
+
+        requested.ShouldBeFalse("no supervisor → no recycle path; action logged only");
+    }
+
+    [Theory]
+    [InlineData(DriverTier.A)]
+    [InlineData(DriverTier.B)]
+    [InlineData(DriverTier.C)]
+    public async Task SoftBreach_NeverRequestsRecycle(DriverTier tier)
+    {
+        var supervisor = new FakeSupervisor();
+        var recycle = new MemoryRecycle(tier, supervisor, NullLogger<MemoryRecycle>.Instance);
+
+        var requested = await recycle.HandleAsync(MemoryTrackingAction.SoftBreach, 1_000_000_000, CancellationToken.None);
+
+        requested.ShouldBeFalse("soft-breach is surface-only at every tier");
+        supervisor.RecycleCount.ShouldBe(0);
+    }
+
+    [Theory]
+    [InlineData(MemoryTrackingAction.None)]
+    [InlineData(MemoryTrackingAction.Warming)]
+    public async Task NonBreachActions_NoOp(MemoryTrackingAction action)
+    {
+        var supervisor = new FakeSupervisor();
+        var recycle = new MemoryRecycle(DriverTier.C, supervisor, NullLogger<MemoryRecycle>.Instance);
+
+        var requested = await recycle.HandleAsync(action, 100_000_000, CancellationToken.None);
+
+        requested.ShouldBeFalse();
+        supervisor.RecycleCount.ShouldBe(0);
+    }
+
+    private sealed class FakeSupervisor : IDriverSupervisor
+    {
+        public string DriverInstanceId => "fake-tier-c";
+        public int RecycleCount { get; private set; }
+        public string? LastReason { get; private set; }
+
+        public Task RecycleAsync(string reason, CancellationToken cancellationToken)
+        {
+            RecycleCount++;
+            LastReason = reason;
+            return Task.CompletedTask;
+        }
+    }
+}

tests/ZB.MOM.WW.OtOpcUa.Core.Tests/Stability/MemoryTrackingTests.cs

@@ -0,0 +1,119 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Stability;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Stability;
+
+[Trait("Category", "Unit")]
+public sealed class MemoryTrackingTests
+{
+    private static readonly DateTime T0 = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc);
+
+    [Fact]
+    public void WarmingUp_Returns_Warming_UntilWindowElapses()
+    {
+        var tracker = new MemoryTracking(DriverTier.A, TimeSpan.FromMinutes(5));
+
+        tracker.Sample(100_000_000, T0).ShouldBe(MemoryTrackingAction.Warming);
+        tracker.Sample(105_000_000, T0.AddMinutes(1)).ShouldBe(MemoryTrackingAction.Warming);
+        tracker.Sample(102_000_000, T0.AddMinutes(4.9)).ShouldBe(MemoryTrackingAction.Warming);
+
+        tracker.Phase.ShouldBe(TrackingPhase.WarmingUp);
+        tracker.BaselineBytes.ShouldBe(0);
+    }
+
+    [Fact]
+    public void WindowElapsed_CapturesBaselineAsMedian_AndTransitionsToSteady()
+    {
+        var tracker = new MemoryTracking(DriverTier.A, TimeSpan.FromMinutes(5));
+
+        tracker.Sample(100_000_000, T0);
+        tracker.Sample(200_000_000, T0.AddMinutes(1));
+        tracker.Sample(150_000_000, T0.AddMinutes(2));
+        var first = tracker.Sample(150_000_000, T0.AddMinutes(5));
+
+        tracker.Phase.ShouldBe(TrackingPhase.Steady);
+        tracker.BaselineBytes.ShouldBe(150_000_000L, "median of 4 samples [100, 200, 150, 150] = (150+150)/2 = 150");
+        first.ShouldBe(MemoryTrackingAction.None, "150 MB is the baseline itself, well under soft threshold");
+    }
+
+    [Theory]
+    [InlineData(DriverTier.A, 3, 50)]
+    [InlineData(DriverTier.B, 3, 100)]
+    [InlineData(DriverTier.C, 2, 500)]
+    public void GetTierConstants_MatchesDecision146(DriverTier tier, int expectedMultiplier, long expectedFloorMB)
+    {
+        var (multiplier, floor) = MemoryTracking.GetTierConstants(tier);
+        multiplier.ShouldBe(expectedMultiplier);
+        floor.ShouldBe(expectedFloorMB * 1024 * 1024);
+    }
+
+    [Fact]
+    public void SoftThreshold_UsesMax_OfMultiplierAndFloor_SmallBaseline()
+    {
+        // Tier A: mult=3, floor=50 MB. Baseline 10 MB → 3×10=30 MB < 10+50=60 MB → floor wins.
+        var tracker = WarmupWithBaseline(DriverTier.A, 10L * 1024 * 1024);
+        tracker.SoftThresholdBytes.ShouldBe(60L * 1024 * 1024);
+    }
+
+    [Fact]
+    public void SoftThreshold_UsesMax_OfMultiplierAndFloor_LargeBaseline()
+    {
+        // Tier A: mult=3, floor=50 MB. Baseline 200 MB → 3×200=600 MB > 200+50=250 MB → multiplier wins.
+        var tracker = WarmupWithBaseline(DriverTier.A, 200L * 1024 * 1024);
+        tracker.SoftThresholdBytes.ShouldBe(600L * 1024 * 1024);
+    }
+
+    [Fact]
+    public void HardThreshold_IsTwiceSoft()
+    {
+        var tracker = WarmupWithBaseline(DriverTier.B, 200L * 1024 * 1024);
+        tracker.HardThresholdBytes.ShouldBe(tracker.SoftThresholdBytes * 2);
+    }
+
+    [Fact]
+    public void Sample_Below_Soft_Returns_None()
+    {
+        var tracker = WarmupWithBaseline(DriverTier.A, 100L * 1024 * 1024);
+
+        tracker.Sample(200L * 1024 * 1024, T0.AddMinutes(10)).ShouldBe(MemoryTrackingAction.None);
+    }
+
+    [Fact]
+    public void Sample_AtSoft_Returns_SoftBreach()
+    {
+        // Tier A, baseline 200 MB → soft = 600 MB. Sample exactly at soft.
+        var tracker = WarmupWithBaseline(DriverTier.A, 200L * 1024 * 1024);
+
+        tracker.Sample(tracker.SoftThresholdBytes, T0.AddMinutes(10))
+            .ShouldBe(MemoryTrackingAction.SoftBreach);
+    }
+
+    [Fact]
+    public void Sample_AtHard_Returns_HardBreach()
+    {
+        var tracker = WarmupWithBaseline(DriverTier.A, 200L * 1024 * 1024);
+
+        tracker.Sample(tracker.HardThresholdBytes, T0.AddMinutes(10))
+            .ShouldBe(MemoryTrackingAction.HardBreach);
+    }
+
+    [Fact]
+    public void Sample_AboveHard_Returns_HardBreach()
+    {
+        var tracker = WarmupWithBaseline(DriverTier.A, 200L * 1024 * 1024);
+
+        tracker.Sample(tracker.HardThresholdBytes + 100_000_000, T0.AddMinutes(10))
+            .ShouldBe(MemoryTrackingAction.HardBreach);
+    }
+
+    private static MemoryTracking WarmupWithBaseline(DriverTier tier, long baseline)
+    {
+        var tracker = new MemoryTracking(tier, TimeSpan.FromMinutes(5));
+        tracker.Sample(baseline, T0);
+        tracker.Sample(baseline, T0.AddMinutes(5));
+        tracker.BaselineBytes.ShouldBe(baseline);
+        return tracker;
+    }
+}

tests/ZB.MOM.WW.OtOpcUa.Core.Tests/Stability/ScheduledRecycleSchedulerTests.cs

@@ -0,0 +1,101 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Stability;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Stability;
+
+[Trait("Category", "Unit")]
+public sealed class ScheduledRecycleSchedulerTests
+{
+    private static readonly DateTime T0 = new(2026, 4, 19, 0, 0, 0, DateTimeKind.Utc);
+    private static readonly TimeSpan Weekly = TimeSpan.FromDays(7);
+
+    [Theory]
+    [InlineData(DriverTier.A)]
+    [InlineData(DriverTier.B)]
+    public void TierAOrB_Ctor_Throws(DriverTier tier)
+    {
+        var supervisor = new FakeSupervisor();
+        Should.Throw<ArgumentException>(() => new ScheduledRecycleScheduler(
+            tier, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance));
+    }
+
+    [Fact]
+    public void ZeroOrNegativeInterval_Throws()
+    {
+        var supervisor = new FakeSupervisor();
+        Should.Throw<ArgumentException>(() => new ScheduledRecycleScheduler(
+            DriverTier.C, TimeSpan.Zero, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance));
+        Should.Throw<ArgumentException>(() => new ScheduledRecycleScheduler(
+            DriverTier.C, TimeSpan.FromSeconds(-1), T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance));
+    }
+
+    [Fact]
+    public async Task Tick_BeforeNextRecycle_NoOp()
+    {
+        var supervisor = new FakeSupervisor();
+        var sch = new ScheduledRecycleScheduler(DriverTier.C, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
+
+        var fired = await sch.TickAsync(T0 + TimeSpan.FromDays(6), CancellationToken.None);
+
+        fired.ShouldBeFalse();
+        supervisor.RecycleCount.ShouldBe(0);
+    }
+
+    [Fact]
+    public async Task Tick_AtOrAfterNextRecycle_FiresOnce_AndAdvances()
+    {
+        var supervisor = new FakeSupervisor();
+        var sch = new ScheduledRecycleScheduler(DriverTier.C, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
+
+        var fired = await sch.TickAsync(T0 + Weekly + TimeSpan.FromMinutes(1), CancellationToken.None);
+
+        fired.ShouldBeTrue();
+        supervisor.RecycleCount.ShouldBe(1);
+        sch.NextRecycleUtc.ShouldBe(T0 + Weekly + Weekly);
+    }
+
+    [Fact]
+    public async Task RequestRecycleNow_Fires_Immediately_WithoutAdvancingSchedule()
+    {
+        var supervisor = new FakeSupervisor();
+        var sch = new ScheduledRecycleScheduler(DriverTier.C, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
+        var nextBefore = sch.NextRecycleUtc;
+
+        await sch.RequestRecycleNowAsync("memory hard-breach", CancellationToken.None);
+
+        supervisor.RecycleCount.ShouldBe(1);
+        supervisor.LastReason.ShouldBe("memory hard-breach");
+        sch.NextRecycleUtc.ShouldBe(nextBefore, "ad-hoc recycle doesn't shift the cron schedule");
+    }
+
+    [Fact]
+    public async Task MultipleFires_AcrossTicks_AdvanceOneIntervalEach()
+    {
+        var supervisor = new FakeSupervisor();
+        var sch = new ScheduledRecycleScheduler(DriverTier.C, TimeSpan.FromDays(1), T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
+
+        await sch.TickAsync(T0 + TimeSpan.FromDays(1) + TimeSpan.FromHours(1), CancellationToken.None);
+        await sch.TickAsync(T0 + TimeSpan.FromDays(2) + TimeSpan.FromHours(1), CancellationToken.None);
+        await sch.TickAsync(T0 + TimeSpan.FromDays(3) + TimeSpan.FromHours(1), CancellationToken.None);
+
+        supervisor.RecycleCount.ShouldBe(3);
+        sch.NextRecycleUtc.ShouldBe(T0 + TimeSpan.FromDays(4));
+    }
+
+    private sealed class FakeSupervisor : IDriverSupervisor
+    {
+        public string DriverInstanceId => "tier-c-fake";
+        public int RecycleCount { get; private set; }
+        public string? LastReason { get; private set; }
+
+        public Task RecycleAsync(string reason, CancellationToken cancellationToken)
+        {
+            RecycleCount++;
+            LastReason = reason;
+            return Task.CompletedTask;
+        }
+    }
+}

tests/ZB.MOM.WW.OtOpcUa.Core.Tests/Stability/WedgeDetectorTests.cs

@@ -0,0 +1,112 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Stability;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Stability;
+
+[Trait("Category", "Unit")]
+public sealed class WedgeDetectorTests
+{
+    private static readonly DateTime Now = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc);
+    private static readonly TimeSpan Threshold = TimeSpan.FromSeconds(120);
+
+    [Fact]
+    public void SubSixtySecondThreshold_ClampsToSixty()
+    {
+        var detector = new WedgeDetector(TimeSpan.FromSeconds(10));
+        detector.Threshold.ShouldBe(TimeSpan.FromSeconds(60));
+    }
+
+    [Fact]
+    public void Unhealthy_Driver_AlwaysNotApplicable()
+    {
+        var detector = new WedgeDetector(Threshold);
+        var demand = new DemandSignal(BulkheadDepth: 5, ActiveMonitoredItems: 10, QueuedHistoryReads: 0, LastProgressUtc: Now.AddMinutes(-10));
+
+        detector.Classify(DriverState.Faulted, demand, Now).ShouldBe(WedgeVerdict.NotApplicable);
+        detector.Classify(DriverState.Degraded, demand, Now).ShouldBe(WedgeVerdict.NotApplicable);
+        detector.Classify(DriverState.Initializing, demand, Now).ShouldBe(WedgeVerdict.NotApplicable);
+    }
+
+    [Fact]
+    public void Idle_Subscription_Only_StaysIdle()
+    {
+        // Idle driver: bulkhead 0, monitored items 0, no history reads queued.
+        // Even if LastProgressUtc is ancient, the verdict is Idle, not Faulted.
+        var detector = new WedgeDetector(Threshold);
+        var demand = new DemandSignal(0, 0, 0, Now.AddHours(-12));
+
+        detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Idle);
+    }
+
+    [Fact]
+    public void PendingWork_WithRecentProgress_StaysHealthy()
+    {
+        var detector = new WedgeDetector(Threshold);
+        var demand = new DemandSignal(BulkheadDepth: 2, ActiveMonitoredItems: 0, QueuedHistoryReads: 0, LastProgressUtc: Now.AddSeconds(-30));
+
+        detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Healthy);
+    }
+
+    [Fact]
+    public void PendingWork_WithStaleProgress_IsFaulted()
+    {
+        var detector = new WedgeDetector(Threshold);
+        var demand = new DemandSignal(BulkheadDepth: 2, ActiveMonitoredItems: 0, QueuedHistoryReads: 0, LastProgressUtc: Now.AddMinutes(-5));
+
+        detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Faulted);
+    }
+
+    [Fact]
+    public void MonitoredItems_Active_ButNoRecentPublish_IsFaulted()
+    {
+        // Subscription-only driver with live MonitoredItems but no publish progress within threshold
+        // is a real wedge — this is the case the previous "no successful Read" formulation used
+        // to miss (no reads ever happen).
+        var detector = new WedgeDetector(Threshold);
+        var demand = new DemandSignal(BulkheadDepth: 0, ActiveMonitoredItems: 5, QueuedHistoryReads: 0, LastProgressUtc: Now.AddMinutes(-10));
+
+        detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Faulted);
+    }
+
+    [Fact]
+    public void MonitoredItems_Active_WithFreshPublish_StaysHealthy()
+    {
+        var detector = new WedgeDetector(Threshold);
+        var demand = new DemandSignal(BulkheadDepth: 0, ActiveMonitoredItems: 5, QueuedHistoryReads: 0, LastProgressUtc: Now.AddSeconds(-10));
+
+        detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Healthy);
+    }
+
+    [Fact]
+    public void HistoryBackfill_SlowButMakingProgress_StaysHealthy()
+    {
+        // Slow historian backfill — QueuedHistoryReads > 0 but progress advances within threshold.
+        var detector = new WedgeDetector(Threshold);
+        var demand = new DemandSignal(BulkheadDepth: 0, ActiveMonitoredItems: 0, QueuedHistoryReads: 50, LastProgressUtc: Now.AddSeconds(-60));
+
+        detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Healthy);
+    }
+
+    [Fact]
+    public void WriteOnlyBurst_StaysIdle_WhenBulkheadEmpty()
+    {
+        // A write-only driver that just finished a burst: bulkhead drained, no subscriptions, no
+        // history reads. Idle — the previous formulation would have faulted here because no
+        // reads were succeeding even though the driver is perfectly healthy.
+        var detector = new WedgeDetector(Threshold);
+        var demand = new DemandSignal(0, 0, 0, Now.AddMinutes(-30));
+
+        detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Idle);
+    }
+
+    [Fact]
+    public void DemandSignal_HasPendingWork_TrueForAnyNonZeroCounter()
+    {
+        new DemandSignal(1, 0, 0, Now).HasPendingWork.ShouldBeTrue();
+        new DemandSignal(0, 1, 0, Now).HasPendingWork.ShouldBeTrue();
+        new DemandSignal(0, 0, 1, Now).HasPendingWork.ShouldBeTrue();
+        new DemandSignal(0, 0, 0, Now).HasPendingWork.ShouldBeFalse();
+    }
+}