Phase 6.1 Stream B (partial) - Tier registry invariant + MemoryTracking with hybrid formula #79

Merged
dohertj2 merged 2 commits from phase-6-1-stream-b-stability into v2 2026-04-19 08:05:04 -04:00
11 changed files with 844 additions and 3 deletions

View File

@@ -69,12 +69,20 @@ public sealed class DriverTypeRegistry
/// <param name="DriverConfigJsonSchema">JSON Schema (Draft 2020-12) the driver's <c>DriverConfig</c> column must validate against.</param>
/// <param name="DeviceConfigJsonSchema">JSON Schema for <c>DeviceConfig</c> (multi-device drivers); null if the driver has no device layer.</param>
/// <param name="TagConfigJsonSchema">JSON Schema for <c>TagConfig</c>; required for every driver since every driver has tags.</param>
/// <param name="Tier">
/// Stability tier per <c>docs/v2/driver-stability.md</c> §2-4 and <c>docs/v2/plan.md</c>
/// decisions #63-74. Drives the shared resilience pipeline defaults
/// (<see cref="Tier"/> × capability → <c>CapabilityPolicy</c>), the <c>MemoryTracking</c>
/// hybrid-formula constants, and whether process-level <c>MemoryRecycle</c> / scheduled-
/// recycle protections apply (Tier C only). Every registered driver type must declare one.
/// </param>
public sealed record DriverTypeMetadata(
string TypeName,
NamespaceKindCompatibility AllowedNamespaceKinds,
string DriverConfigJsonSchema,
string? DeviceConfigJsonSchema,
string TagConfigJsonSchema);
string TagConfigJsonSchema,
DriverTier Tier);
/// <summary>Bitmask of namespace kinds a driver type may populate. Per decision #111.</summary>
[Flags]

View File

@@ -0,0 +1,26 @@
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
/// <summary>
/// Process-level supervisor contract a Tier C driver's out-of-process topology provides
/// (e.g. <c>Driver.Galaxy.Proxy/Supervisor/</c>). Concerns: restart the Host process when a
/// hard fault is detected (memory breach, wedge, scheduled recycle window).
/// </summary>
/// <remarks>
/// Per <c>docs/v2/plan.md</c> decisions #68, #73-74, and #145. Tier A/B drivers do NOT have
/// a supervisor because they run in-process — recycling would kill every OPC UA session and
/// every co-hosted driver. The Core.Stability layer only invokes this interface for Tier C
/// instances after asserting the tier via <see cref="DriverTypeMetadata.Tier"/>.
/// </remarks>
public interface IDriverSupervisor
{
/// <summary>Driver instance this supervisor governs.</summary>
string DriverInstanceId { get; }
/// <summary>
/// Request the supervisor to recycle (terminate + restart) the Host process. Implementations
/// are expected to be idempotent under repeat calls during an in-flight recycle.
/// </summary>
/// <param name="reason">Human-readable reason — flows into the supervisor's logs.</param>
/// <param name="cancellationToken">Cancels the recycle request; an in-flight restart is not interrupted.</param>
Task RecycleAsync(string reason, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,65 @@
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Core.Stability;
/// <summary>
/// Tier C only process-recycle companion to <see cref="MemoryTracking"/>. On a
/// <see cref="MemoryTrackingAction.HardBreach"/> signal, invokes the supplied
/// <see cref="IDriverSupervisor"/> to restart the out-of-process Host.
/// </summary>
/// <remarks>
/// Per <c>docs/v2/plan.md</c> decisions #74 and #145. Tier A/B hard-breach on an in-process
/// driver would kill every OPC UA session and every co-hosted driver, so for Tier A/B this
/// class logs a <b>promotion-to-Tier-C recommendation</b> and does NOT invoke any supervisor.
/// A future tier-migration workflow acts on the recommendation.
/// </remarks>
public sealed class MemoryRecycle
{
private readonly DriverTier _tier;
private readonly IDriverSupervisor? _supervisor;
private readonly ILogger<MemoryRecycle> _logger;
public MemoryRecycle(DriverTier tier, IDriverSupervisor? supervisor, ILogger<MemoryRecycle> logger)
{
_tier = tier;
_supervisor = supervisor;
_logger = logger;
}
/// <summary>
/// Handle a <see cref="MemoryTracking"/> classification for the driver. For Tier C with a
/// wired supervisor, <c>HardBreach</c> triggers <see cref="IDriverSupervisor.RecycleAsync"/>.
/// All other combinations are no-ops with respect to process state (soft breaches + Tier A/B
/// hard breaches just log).
/// </summary>
/// <returns>True when a recycle was requested; false otherwise.</returns>
public async Task<bool> HandleAsync(MemoryTrackingAction action, long footprintBytes, CancellationToken cancellationToken)
{
switch (action)
{
case MemoryTrackingAction.SoftBreach:
_logger.LogWarning(
"Memory soft-breach on driver {DriverId}: footprint={Footprint:N0} bytes, tier={Tier}. Surfaced to Admin; no action.",
_supervisor?.DriverInstanceId ?? "(unknown)", footprintBytes, _tier);
return false;
case MemoryTrackingAction.HardBreach when _tier == DriverTier.C && _supervisor is not null:
_logger.LogError(
"Memory hard-breach on Tier C driver {DriverId}: footprint={Footprint:N0} bytes. Requesting supervisor recycle.",
_supervisor.DriverInstanceId, footprintBytes);
await _supervisor.RecycleAsync($"Memory hard-breach: {footprintBytes} bytes", cancellationToken).ConfigureAwait(false);
return true;
case MemoryTrackingAction.HardBreach:
_logger.LogError(
"Memory hard-breach on Tier {Tier} in-process driver {DriverId}: footprint={Footprint:N0} bytes. " +
"Recommending promotion to Tier C; NOT auto-killing (decisions #74, #145).",
_tier, _supervisor?.DriverInstanceId ?? "(unknown)", footprintBytes);
return false;
default:
return false;
}
}
}

View File

@@ -0,0 +1,136 @@
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Core.Stability;
/// <summary>
/// Tier-agnostic memory-footprint tracker. Captures the post-initialize <b>baseline</b>
/// from the first samples after <c>IDriver.InitializeAsync</c>, then classifies each
/// subsequent sample against a hybrid soft/hard threshold per
/// <c>docs/v2/plan.md</c> decision #146 — <c>soft = max(multiplier × baseline, baseline + floor)</c>,
/// <c>hard = 2 × soft</c>.
/// </summary>
/// <remarks>
/// <para>Per decision #145, this tracker <b>never kills a process</b>. Soft and hard breaches
/// log + surface to the Admin UI via <c>DriverInstanceResilienceStatus</c>. The matching
/// process-level recycle protection lives in a separate <c>MemoryRecycle</c> that activates
/// for Tier C drivers only (where the driver runs out-of-process behind a supervisor that
/// can safely restart it without tearing down the OPC UA session or co-hosted in-proc
/// drivers).</para>
///
/// <para>Baseline capture: the tracker starts in <see cref="TrackingPhase.WarmingUp"/> for
/// <see cref="BaselineWindow"/> (default 5 min). During that window samples are collected;
/// the baseline is computed as the median once the window elapses. Before that point every
/// classification returns <see cref="MemoryTrackingAction.Warming"/>.</para>
/// </remarks>
public sealed class MemoryTracking
{
private readonly DriverTier _tier;
private readonly TimeSpan _baselineWindow;
private readonly List<long> _warmupSamples = [];
private long _baselineBytes;
private TrackingPhase _phase = TrackingPhase.WarmingUp;
private DateTime? _warmupStartUtc;
/// <summary>Tier-default multiplier/floor constants per decision #146.</summary>
public static (int Multiplier, long FloorBytes) GetTierConstants(DriverTier tier) => tier switch
{
DriverTier.A => (Multiplier: 3, FloorBytes: 50L * 1024 * 1024),
DriverTier.B => (Multiplier: 3, FloorBytes: 100L * 1024 * 1024),
DriverTier.C => (Multiplier: 2, FloorBytes: 500L * 1024 * 1024),
_ => throw new ArgumentOutOfRangeException(nameof(tier), tier, $"No memory-tracking constants defined for tier {tier}."),
};
/// <summary>Window over which post-init samples are collected to compute the baseline.</summary>
public TimeSpan BaselineWindow => _baselineWindow;
/// <summary>Current phase: <see cref="TrackingPhase.WarmingUp"/> or <see cref="TrackingPhase.Steady"/>.</summary>
public TrackingPhase Phase => _phase;
/// <summary>Captured baseline; 0 until warmup completes.</summary>
public long BaselineBytes => _baselineBytes;
/// <summary>Effective soft threshold (zero while warming up).</summary>
public long SoftThresholdBytes => _baselineBytes == 0 ? 0 : ComputeSoft(_tier, _baselineBytes);
/// <summary>Effective hard threshold = 2 × soft (zero while warming up).</summary>
public long HardThresholdBytes => _baselineBytes == 0 ? 0 : ComputeSoft(_tier, _baselineBytes) * 2;
public MemoryTracking(DriverTier tier, TimeSpan? baselineWindow = null)
{
_tier = tier;
_baselineWindow = baselineWindow ?? TimeSpan.FromMinutes(5);
}
/// <summary>
/// Submit a memory-footprint sample. Returns the action the caller should surface.
/// During warmup, always returns <see cref="MemoryTrackingAction.Warming"/> and accumulates
/// samples; once the window elapses the first steady-phase sample triggers baseline capture
/// (median of warmup samples).
/// </summary>
public MemoryTrackingAction Sample(long footprintBytes, DateTime utcNow)
{
if (_phase == TrackingPhase.WarmingUp)
{
_warmupStartUtc ??= utcNow;
_warmupSamples.Add(footprintBytes);
if (utcNow - _warmupStartUtc.Value >= _baselineWindow && _warmupSamples.Count > 0)
{
_baselineBytes = ComputeMedian(_warmupSamples);
_phase = TrackingPhase.Steady;
}
else
{
return MemoryTrackingAction.Warming;
}
}
if (footprintBytes >= HardThresholdBytes) return MemoryTrackingAction.HardBreach;
if (footprintBytes >= SoftThresholdBytes) return MemoryTrackingAction.SoftBreach;
return MemoryTrackingAction.None;
}
private static long ComputeSoft(DriverTier tier, long baseline)
{
var (multiplier, floor) = GetTierConstants(tier);
return Math.Max(multiplier * baseline, baseline + floor);
}
private static long ComputeMedian(List<long> samples)
{
var sorted = samples.Order().ToArray();
var mid = sorted.Length / 2;
return sorted.Length % 2 == 1
? sorted[mid]
: (sorted[mid - 1] + sorted[mid]) / 2;
}
}
/// <summary>Phase of a <see cref="MemoryTracking"/> lifecycle.</summary>
public enum TrackingPhase
{
/// <summary>Collecting post-init samples; baseline not yet computed.</summary>
WarmingUp,
/// <summary>Baseline captured; every sample classified against soft/hard thresholds.</summary>
Steady,
}
/// <summary>Classification the tracker returns per sample.</summary>
public enum MemoryTrackingAction
{
/// <summary>Baseline not yet captured; sample collected, no threshold check.</summary>
Warming,
/// <summary>Below soft threshold.</summary>
None,
/// <summary>Between soft and hard thresholds — log + surface, no action.</summary>
SoftBreach,
/// <summary>
/// ≥ hard threshold. Log + surface + (Tier C only, via <c>MemoryRecycle</c>) request
/// process recycle via the driver supervisor. Tier A/B breach never invokes any
/// kill path per decisions #145 and #74.
/// </summary>
HardBreach,
}

View File

@@ -0,0 +1,86 @@
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Core.Stability;
/// <summary>
/// Tier C opt-in periodic-recycle driver per <c>docs/v2/plan.md</c> decision #67.
/// A tick method advanced by the caller (fed by a background timer in prod; by test clock
/// in unit tests) decides whether the configured interval has elapsed and, if so, drives the
/// supplied <see cref="IDriverSupervisor"/> to recycle the Host.
/// </summary>
/// <remarks>
/// Tier A/B drivers MUST NOT use this class — scheduled recycle for in-process drivers would
/// kill every OPC UA session and every co-hosted driver. The ctor throws when constructed
/// with any tier other than C to make the misuse structurally impossible.
///
/// <para>Keeps no background thread of its own — callers invoke <see cref="TickAsync"/> on
/// their ambient scheduler tick (Phase 6.1 Stream C's health-endpoint host runs one). That
/// decouples the unit under test from wall-clock time and thread-pool scheduling.</para>
/// </remarks>
public sealed class ScheduledRecycleScheduler
{
private readonly TimeSpan _recycleInterval;
private readonly IDriverSupervisor _supervisor;
private readonly ILogger<ScheduledRecycleScheduler> _logger;
private DateTime _nextRecycleUtc;
/// <summary>
/// Construct the scheduler for a Tier C driver. Throws if <paramref name="tier"/> isn't C.
/// </summary>
/// <param name="tier">Driver tier; must be <see cref="DriverTier.C"/>.</param>
/// <param name="recycleInterval">Interval between recycles (e.g. 7 days).</param>
/// <param name="startUtc">Anchor time; next recycle fires at <paramref name="startUtc"/> + <paramref name="recycleInterval"/>.</param>
/// <param name="supervisor">Supervisor that performs the actual recycle.</param>
/// <param name="logger">Diagnostic sink.</param>
public ScheduledRecycleScheduler(
DriverTier tier,
TimeSpan recycleInterval,
DateTime startUtc,
IDriverSupervisor supervisor,
ILogger<ScheduledRecycleScheduler> logger)
{
if (tier != DriverTier.C)
throw new ArgumentException(
$"ScheduledRecycleScheduler is Tier C only (got {tier}). " +
"In-process drivers must not use scheduled recycle; see decisions #74 and #145.",
nameof(tier));
if (recycleInterval <= TimeSpan.Zero)
throw new ArgumentException("RecycleInterval must be positive.", nameof(recycleInterval));
_recycleInterval = recycleInterval;
_supervisor = supervisor;
_logger = logger;
_nextRecycleUtc = startUtc + recycleInterval;
}
/// <summary>Next scheduled recycle UTC. Advances by <see cref="RecycleInterval"/> on each fire.</summary>
public DateTime NextRecycleUtc => _nextRecycleUtc;
/// <summary>Recycle interval this scheduler was constructed with.</summary>
public TimeSpan RecycleInterval => _recycleInterval;
/// <summary>
/// Tick the scheduler forward. If <paramref name="utcNow"/> is past
/// <see cref="NextRecycleUtc"/>, requests a recycle from the supervisor and advances
/// <see cref="NextRecycleUtc"/> by exactly one interval. Returns true when a recycle fired.
/// </summary>
public async Task<bool> TickAsync(DateTime utcNow, CancellationToken cancellationToken)
{
if (utcNow < _nextRecycleUtc)
return false;
_logger.LogInformation(
"Scheduled recycle due for Tier C driver {DriverId} at {Now:o}; advancing next to {Next:o}.",
_supervisor.DriverInstanceId, utcNow, _nextRecycleUtc + _recycleInterval);
await _supervisor.RecycleAsync("Scheduled periodic recycle", cancellationToken).ConfigureAwait(false);
_nextRecycleUtc += _recycleInterval;
return true;
}
/// <summary>Request an immediate recycle outside the schedule (e.g. MemoryRecycle hard-breach escalation).</summary>
public Task RequestRecycleNowAsync(string reason, CancellationToken cancellationToken) =>
_supervisor.RecycleAsync(reason, cancellationToken);
}

View File

@@ -0,0 +1,81 @@
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Core.Stability;
/// <summary>
/// Demand-aware driver-wedge detector per <c>docs/v2/plan.md</c> decision #147.
/// Flips a driver to <see cref="WedgeVerdict.Faulted"/> only when BOTH of the following hold:
/// (a) there is pending work outstanding, AND (b) no progress has been observed for longer
/// than <see cref="Threshold"/>. Idle drivers, write-only burst drivers, and subscription-only
/// drivers whose signals don't arrive regularly all stay Healthy.
/// </summary>
/// <remarks>
/// <para>Pending work signal is supplied by the caller via <see cref="DemandSignal"/>:
/// non-zero Polly bulkhead depth, ≥1 active MonitoredItem, or ≥1 queued historian read
/// each qualifies. The detector itself is state-light: all it remembers is the last
/// <c>LastProgressUtc</c> it saw and the last wedge verdict. No history buffer.</para>
///
/// <para>Default threshold per plan: <c>5 × PublishingInterval</c>, with a minimum of 60 s.
/// Concrete values are driver-agnostic and configured per-instance by the caller.</para>
/// </remarks>
public sealed class WedgeDetector
{
/// <summary>Wedge-detection threshold; pass &lt; 60 s and the detector clamps to 60 s.</summary>
public TimeSpan Threshold { get; }
/// <summary>Whether the driver reported itself <see cref="DriverState.Healthy"/> at construction.</summary>
public WedgeDetector(TimeSpan threshold)
{
Threshold = threshold < TimeSpan.FromSeconds(60) ? TimeSpan.FromSeconds(60) : threshold;
}
/// <summary>
/// Classify the current state against the demand signal. Does not retain state across
/// calls — each call is self-contained; the caller owns the <c>LastProgressUtc</c> clock.
/// </summary>
public WedgeVerdict Classify(DriverState state, DemandSignal demand, DateTime utcNow)
{
if (state != DriverState.Healthy)
return WedgeVerdict.NotApplicable;
if (!demand.HasPendingWork)
return WedgeVerdict.Idle;
var sinceProgress = utcNow - demand.LastProgressUtc;
return sinceProgress > Threshold ? WedgeVerdict.Faulted : WedgeVerdict.Healthy;
}
}
/// <summary>
/// Caller-supplied demand snapshot. All three counters are OR'd — any non-zero means work
/// is outstanding, which is the trigger for checking the <see cref="LastProgressUtc"/> clock.
/// </summary>
/// <param name="BulkheadDepth">Polly bulkhead depth (in-flight capability calls).</param>
/// <param name="ActiveMonitoredItems">Number of live OPC UA MonitoredItems bound to this driver.</param>
/// <param name="QueuedHistoryReads">Pending historian-read requests the driver owes the server.</param>
/// <param name="LastProgressUtc">Last time the driver reported a successful unit of work (read, subscribe-ack, publish).</param>
public readonly record struct DemandSignal(
int BulkheadDepth,
int ActiveMonitoredItems,
int QueuedHistoryReads,
DateTime LastProgressUtc)
{
/// <summary>True when any of the three counters is &gt; 0.</summary>
public bool HasPendingWork => BulkheadDepth > 0 || ActiveMonitoredItems > 0 || QueuedHistoryReads > 0;
}
/// <summary>Outcome of a single <see cref="WedgeDetector.Classify"/> call.</summary>
public enum WedgeVerdict
{
/// <summary>Driver wasn't Healthy to begin with — wedge detection doesn't apply.</summary>
NotApplicable,
/// <summary>Driver claims Healthy + no pending work → stays Healthy.</summary>
Idle,
/// <summary>Driver claims Healthy + has pending work + has made progress within the threshold → stays Healthy.</summary>
Healthy,
/// <summary>Driver claims Healthy + has pending work + has NOT made progress within the threshold → wedged.</summary>
Faulted,
}

View File

@@ -7,11 +7,13 @@ public sealed class DriverTypeRegistryTests
{
private static DriverTypeMetadata SampleMetadata(
string typeName = "Modbus",
NamespaceKindCompatibility allowed = NamespaceKindCompatibility.Equipment) =>
NamespaceKindCompatibility allowed = NamespaceKindCompatibility.Equipment,
DriverTier tier = DriverTier.B) =>
new(typeName, allowed,
DriverConfigJsonSchema: "{\"type\": \"object\"}",
DeviceConfigJsonSchema: "{\"type\": \"object\"}",
TagConfigJsonSchema: "{\"type\": \"object\"}");
TagConfigJsonSchema: "{\"type\": \"object\"}",
Tier: tier);
[Fact]
public void Register_ThenGet_RoundTrips()
@@ -24,6 +26,20 @@ public sealed class DriverTypeRegistryTests
registry.Get("Modbus").ShouldBe(metadata);
}
[Theory]
[InlineData(DriverTier.A)]
[InlineData(DriverTier.B)]
[InlineData(DriverTier.C)]
public void Register_Requires_NonNullTier(DriverTier tier)
{
var registry = new DriverTypeRegistry();
var metadata = SampleMetadata(typeName: $"Driver-{tier}", tier: tier);
registry.Register(metadata);
registry.Get(metadata.TypeName).Tier.ShouldBe(tier);
}
[Fact]
public void Get_IsCaseInsensitive()
{

View File

@@ -0,0 +1,91 @@
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Stability;
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Stability;
[Trait("Category", "Unit")]
public sealed class MemoryRecycleTests
{
[Fact]
public async Task TierC_HardBreach_RequestsSupervisorRecycle()
{
var supervisor = new FakeSupervisor();
var recycle = new MemoryRecycle(DriverTier.C, supervisor, NullLogger<MemoryRecycle>.Instance);
var requested = await recycle.HandleAsync(MemoryTrackingAction.HardBreach, 2_000_000_000, CancellationToken.None);
requested.ShouldBeTrue();
supervisor.RecycleCount.ShouldBe(1);
supervisor.LastReason.ShouldContain("hard-breach");
}
[Theory]
[InlineData(DriverTier.A)]
[InlineData(DriverTier.B)]
public async Task InProcessTier_HardBreach_NeverRequestsRecycle(DriverTier tier)
{
var supervisor = new FakeSupervisor();
var recycle = new MemoryRecycle(tier, supervisor, NullLogger<MemoryRecycle>.Instance);
var requested = await recycle.HandleAsync(MemoryTrackingAction.HardBreach, 2_000_000_000, CancellationToken.None);
requested.ShouldBeFalse("Tier A/B hard-breach logs a promotion recommendation only (decisions #74, #145)");
supervisor.RecycleCount.ShouldBe(0);
}
[Fact]
public async Task TierC_WithoutSupervisor_HardBreach_NoOp()
{
var recycle = new MemoryRecycle(DriverTier.C, supervisor: null, NullLogger<MemoryRecycle>.Instance);
var requested = await recycle.HandleAsync(MemoryTrackingAction.HardBreach, 2_000_000_000, CancellationToken.None);
requested.ShouldBeFalse("no supervisor → no recycle path; action logged only");
}
[Theory]
[InlineData(DriverTier.A)]
[InlineData(DriverTier.B)]
[InlineData(DriverTier.C)]
public async Task SoftBreach_NeverRequestsRecycle(DriverTier tier)
{
var supervisor = new FakeSupervisor();
var recycle = new MemoryRecycle(tier, supervisor, NullLogger<MemoryRecycle>.Instance);
var requested = await recycle.HandleAsync(MemoryTrackingAction.SoftBreach, 1_000_000_000, CancellationToken.None);
requested.ShouldBeFalse("soft-breach is surface-only at every tier");
supervisor.RecycleCount.ShouldBe(0);
}
[Theory]
[InlineData(MemoryTrackingAction.None)]
[InlineData(MemoryTrackingAction.Warming)]
public async Task NonBreachActions_NoOp(MemoryTrackingAction action)
{
var supervisor = new FakeSupervisor();
var recycle = new MemoryRecycle(DriverTier.C, supervisor, NullLogger<MemoryRecycle>.Instance);
var requested = await recycle.HandleAsync(action, 100_000_000, CancellationToken.None);
requested.ShouldBeFalse();
supervisor.RecycleCount.ShouldBe(0);
}
private sealed class FakeSupervisor : IDriverSupervisor
{
public string DriverInstanceId => "fake-tier-c";
public int RecycleCount { get; private set; }
public string? LastReason { get; private set; }
public Task RecycleAsync(string reason, CancellationToken cancellationToken)
{
RecycleCount++;
LastReason = reason;
return Task.CompletedTask;
}
}
}

View File

@@ -0,0 +1,119 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Stability;
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Stability;
[Trait("Category", "Unit")]
public sealed class MemoryTrackingTests
{
private static readonly DateTime T0 = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc);
[Fact]
public void WarmingUp_Returns_Warming_UntilWindowElapses()
{
var tracker = new MemoryTracking(DriverTier.A, TimeSpan.FromMinutes(5));
tracker.Sample(100_000_000, T0).ShouldBe(MemoryTrackingAction.Warming);
tracker.Sample(105_000_000, T0.AddMinutes(1)).ShouldBe(MemoryTrackingAction.Warming);
tracker.Sample(102_000_000, T0.AddMinutes(4.9)).ShouldBe(MemoryTrackingAction.Warming);
tracker.Phase.ShouldBe(TrackingPhase.WarmingUp);
tracker.BaselineBytes.ShouldBe(0);
}
[Fact]
public void WindowElapsed_CapturesBaselineAsMedian_AndTransitionsToSteady()
{
var tracker = new MemoryTracking(DriverTier.A, TimeSpan.FromMinutes(5));
tracker.Sample(100_000_000, T0);
tracker.Sample(200_000_000, T0.AddMinutes(1));
tracker.Sample(150_000_000, T0.AddMinutes(2));
var first = tracker.Sample(150_000_000, T0.AddMinutes(5));
tracker.Phase.ShouldBe(TrackingPhase.Steady);
tracker.BaselineBytes.ShouldBe(150_000_000L, "median of 4 samples [100, 200, 150, 150] = (150+150)/2 = 150");
first.ShouldBe(MemoryTrackingAction.None, "150 MB is the baseline itself, well under soft threshold");
}
[Theory]
[InlineData(DriverTier.A, 3, 50)]
[InlineData(DriverTier.B, 3, 100)]
[InlineData(DriverTier.C, 2, 500)]
public void GetTierConstants_MatchesDecision146(DriverTier tier, int expectedMultiplier, long expectedFloorMB)
{
var (multiplier, floor) = MemoryTracking.GetTierConstants(tier);
multiplier.ShouldBe(expectedMultiplier);
floor.ShouldBe(expectedFloorMB * 1024 * 1024);
}
[Fact]
public void SoftThreshold_UsesMax_OfMultiplierAndFloor_SmallBaseline()
{
// Tier A: mult=3, floor=50 MB. Baseline 10 MB → 3×10=30 MB < 10+50=60 MB → floor wins.
var tracker = WarmupWithBaseline(DriverTier.A, 10L * 1024 * 1024);
tracker.SoftThresholdBytes.ShouldBe(60L * 1024 * 1024);
}
[Fact]
public void SoftThreshold_UsesMax_OfMultiplierAndFloor_LargeBaseline()
{
// Tier A: mult=3, floor=50 MB. Baseline 200 MB → 3×200=600 MB > 200+50=250 MB → multiplier wins.
var tracker = WarmupWithBaseline(DriverTier.A, 200L * 1024 * 1024);
tracker.SoftThresholdBytes.ShouldBe(600L * 1024 * 1024);
}
[Fact]
public void HardThreshold_IsTwiceSoft()
{
var tracker = WarmupWithBaseline(DriverTier.B, 200L * 1024 * 1024);
tracker.HardThresholdBytes.ShouldBe(tracker.SoftThresholdBytes * 2);
}
[Fact]
public void Sample_Below_Soft_Returns_None()
{
var tracker = WarmupWithBaseline(DriverTier.A, 100L * 1024 * 1024);
tracker.Sample(200L * 1024 * 1024, T0.AddMinutes(10)).ShouldBe(MemoryTrackingAction.None);
}
[Fact]
public void Sample_AtSoft_Returns_SoftBreach()
{
// Tier A, baseline 200 MB → soft = 600 MB. Sample exactly at soft.
var tracker = WarmupWithBaseline(DriverTier.A, 200L * 1024 * 1024);
tracker.Sample(tracker.SoftThresholdBytes, T0.AddMinutes(10))
.ShouldBe(MemoryTrackingAction.SoftBreach);
}
[Fact]
public void Sample_AtHard_Returns_HardBreach()
{
var tracker = WarmupWithBaseline(DriverTier.A, 200L * 1024 * 1024);
tracker.Sample(tracker.HardThresholdBytes, T0.AddMinutes(10))
.ShouldBe(MemoryTrackingAction.HardBreach);
}
[Fact]
public void Sample_AboveHard_Returns_HardBreach()
{
var tracker = WarmupWithBaseline(DriverTier.A, 200L * 1024 * 1024);
tracker.Sample(tracker.HardThresholdBytes + 100_000_000, T0.AddMinutes(10))
.ShouldBe(MemoryTrackingAction.HardBreach);
}
private static MemoryTracking WarmupWithBaseline(DriverTier tier, long baseline)
{
var tracker = new MemoryTracking(tier, TimeSpan.FromMinutes(5));
tracker.Sample(baseline, T0);
tracker.Sample(baseline, T0.AddMinutes(5));
tracker.BaselineBytes.ShouldBe(baseline);
return tracker;
}
}

View File

@@ -0,0 +1,101 @@
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Stability;
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Stability;
[Trait("Category", "Unit")]
public sealed class ScheduledRecycleSchedulerTests
{
private static readonly DateTime T0 = new(2026, 4, 19, 0, 0, 0, DateTimeKind.Utc);
private static readonly TimeSpan Weekly = TimeSpan.FromDays(7);
[Theory]
[InlineData(DriverTier.A)]
[InlineData(DriverTier.B)]
public void TierAOrB_Ctor_Throws(DriverTier tier)
{
var supervisor = new FakeSupervisor();
Should.Throw<ArgumentException>(() => new ScheduledRecycleScheduler(
tier, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance));
}
[Fact]
public void ZeroOrNegativeInterval_Throws()
{
var supervisor = new FakeSupervisor();
Should.Throw<ArgumentException>(() => new ScheduledRecycleScheduler(
DriverTier.C, TimeSpan.Zero, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance));
Should.Throw<ArgumentException>(() => new ScheduledRecycleScheduler(
DriverTier.C, TimeSpan.FromSeconds(-1), T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance));
}
[Fact]
public async Task Tick_BeforeNextRecycle_NoOp()
{
var supervisor = new FakeSupervisor();
var sch = new ScheduledRecycleScheduler(DriverTier.C, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
var fired = await sch.TickAsync(T0 + TimeSpan.FromDays(6), CancellationToken.None);
fired.ShouldBeFalse();
supervisor.RecycleCount.ShouldBe(0);
}
[Fact]
public async Task Tick_AtOrAfterNextRecycle_FiresOnce_AndAdvances()
{
var supervisor = new FakeSupervisor();
var sch = new ScheduledRecycleScheduler(DriverTier.C, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
var fired = await sch.TickAsync(T0 + Weekly + TimeSpan.FromMinutes(1), CancellationToken.None);
fired.ShouldBeTrue();
supervisor.RecycleCount.ShouldBe(1);
sch.NextRecycleUtc.ShouldBe(T0 + Weekly + Weekly);
}
[Fact]
public async Task RequestRecycleNow_Fires_Immediately_WithoutAdvancingSchedule()
{
var supervisor = new FakeSupervisor();
var sch = new ScheduledRecycleScheduler(DriverTier.C, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
var nextBefore = sch.NextRecycleUtc;
await sch.RequestRecycleNowAsync("memory hard-breach", CancellationToken.None);
supervisor.RecycleCount.ShouldBe(1);
supervisor.LastReason.ShouldBe("memory hard-breach");
sch.NextRecycleUtc.ShouldBe(nextBefore, "ad-hoc recycle doesn't shift the cron schedule");
}
[Fact]
public async Task MultipleFires_AcrossTicks_AdvanceOneIntervalEach()
{
var supervisor = new FakeSupervisor();
var sch = new ScheduledRecycleScheduler(DriverTier.C, TimeSpan.FromDays(1), T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
await sch.TickAsync(T0 + TimeSpan.FromDays(1) + TimeSpan.FromHours(1), CancellationToken.None);
await sch.TickAsync(T0 + TimeSpan.FromDays(2) + TimeSpan.FromHours(1), CancellationToken.None);
await sch.TickAsync(T0 + TimeSpan.FromDays(3) + TimeSpan.FromHours(1), CancellationToken.None);
supervisor.RecycleCount.ShouldBe(3);
sch.NextRecycleUtc.ShouldBe(T0 + TimeSpan.FromDays(4));
}
private sealed class FakeSupervisor : IDriverSupervisor
{
public string DriverInstanceId => "tier-c-fake";
public int RecycleCount { get; private set; }
public string? LastReason { get; private set; }
public Task RecycleAsync(string reason, CancellationToken cancellationToken)
{
RecycleCount++;
LastReason = reason;
return Task.CompletedTask;
}
}
}

View File

@@ -0,0 +1,112 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Stability;
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Stability;
[Trait("Category", "Unit")]
public sealed class WedgeDetectorTests
{
private static readonly DateTime Now = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc);
private static readonly TimeSpan Threshold = TimeSpan.FromSeconds(120);
[Fact]
public void SubSixtySecondThreshold_ClampsToSixty()
{
var detector = new WedgeDetector(TimeSpan.FromSeconds(10));
detector.Threshold.ShouldBe(TimeSpan.FromSeconds(60));
}
[Fact]
public void Unhealthy_Driver_AlwaysNotApplicable()
{
var detector = new WedgeDetector(Threshold);
var demand = new DemandSignal(BulkheadDepth: 5, ActiveMonitoredItems: 10, QueuedHistoryReads: 0, LastProgressUtc: Now.AddMinutes(-10));
detector.Classify(DriverState.Faulted, demand, Now).ShouldBe(WedgeVerdict.NotApplicable);
detector.Classify(DriverState.Degraded, demand, Now).ShouldBe(WedgeVerdict.NotApplicable);
detector.Classify(DriverState.Initializing, demand, Now).ShouldBe(WedgeVerdict.NotApplicable);
}
[Fact]
public void Idle_Subscription_Only_StaysIdle()
{
// Idle driver: bulkhead 0, monitored items 0, no history reads queued.
// Even if LastProgressUtc is ancient, the verdict is Idle, not Faulted.
var detector = new WedgeDetector(Threshold);
var demand = new DemandSignal(0, 0, 0, Now.AddHours(-12));
detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Idle);
}
[Fact]
public void PendingWork_WithRecentProgress_StaysHealthy()
{
var detector = new WedgeDetector(Threshold);
var demand = new DemandSignal(BulkheadDepth: 2, ActiveMonitoredItems: 0, QueuedHistoryReads: 0, LastProgressUtc: Now.AddSeconds(-30));
detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Healthy);
}
[Fact]
public void PendingWork_WithStaleProgress_IsFaulted()
{
var detector = new WedgeDetector(Threshold);
var demand = new DemandSignal(BulkheadDepth: 2, ActiveMonitoredItems: 0, QueuedHistoryReads: 0, LastProgressUtc: Now.AddMinutes(-5));
detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Faulted);
}
[Fact]
public void MonitoredItems_Active_ButNoRecentPublish_IsFaulted()
{
// Subscription-only driver with live MonitoredItems but no publish progress within threshold
// is a real wedge — this is the case the previous "no successful Read" formulation used
// to miss (no reads ever happen).
var detector = new WedgeDetector(Threshold);
var demand = new DemandSignal(BulkheadDepth: 0, ActiveMonitoredItems: 5, QueuedHistoryReads: 0, LastProgressUtc: Now.AddMinutes(-10));
detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Faulted);
}
[Fact]
public void MonitoredItems_Active_WithFreshPublish_StaysHealthy()
{
var detector = new WedgeDetector(Threshold);
var demand = new DemandSignal(BulkheadDepth: 0, ActiveMonitoredItems: 5, QueuedHistoryReads: 0, LastProgressUtc: Now.AddSeconds(-10));
detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Healthy);
}
[Fact]
public void HistoryBackfill_SlowButMakingProgress_StaysHealthy()
{
// Slow historian backfill — QueuedHistoryReads > 0 but progress advances within threshold.
var detector = new WedgeDetector(Threshold);
var demand = new DemandSignal(BulkheadDepth: 0, ActiveMonitoredItems: 0, QueuedHistoryReads: 50, LastProgressUtc: Now.AddSeconds(-60));
detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Healthy);
}
[Fact]
public void WriteOnlyBurst_StaysIdle_WhenBulkheadEmpty()
{
// A write-only driver that just finished a burst: bulkhead drained, no subscriptions, no
// history reads. Idle — the previous formulation would have faulted here because no
// reads were succeeding even though the driver is perfectly healthy.
var detector = new WedgeDetector(Threshold);
var demand = new DemandSignal(0, 0, 0, Now.AddMinutes(-30));
detector.Classify(DriverState.Healthy, demand, Now).ShouldBe(WedgeVerdict.Idle);
}
[Fact]
public void DemandSignal_HasPendingWork_TrueForAnyNonZeroCounter()
{
new DemandSignal(1, 0, 0, Now).HasPendingWork.ShouldBeTrue();
new DemandSignal(0, 1, 0, Now).HasPendingWork.ShouldBeTrue();
new DemandSignal(0, 0, 1, Now).HasPendingWork.ShouldBeTrue();
new DemandSignal(0, 0, 0, Now).HasPendingWork.ShouldBeFalse();
}
}