Closes out Stream B per docs/v2/implementation/phase-6-1-resilience-and-observability.md. Core.Abstractions: - IDriverSupervisor — process-level supervisor contract a Tier C driver's out-of-process topology provides (Galaxy Proxy/Supervisor implements this in a follow-up Driver.Galaxy wiring PR). Concerns: DriverInstanceId + RecycleAsync. Tier A/B drivers don't implement this; Stream B code asserts tier == C before ever calling it. Core.Stability: - MemoryRecycle — companion to MemoryTracking. On HardBreach, invokes the supervisor IFF tier == C AND a supervisor is wired. Tier A/B HardBreach logs a promotion-to-Tier-C recommendation and returns false. Soft/None/Warming never triggers a recycle at any tier. - ScheduledRecycleScheduler — Tier C opt-in periodic recycler per decision #67. Ctor throws for Tier A/B (structural guard — scheduled recycle on an in-process driver would kill every OPC UA session and every co-hosted driver). TickAsync(now) advances the schedule by one interval per fire; RequestRecycleNowAsync drives an ad-hoc recycle without shifting the cron. - WedgeDetector — demand-aware per decision #147. Classify(state, demand, now) returns: * NotApplicable when driver state != Healthy * Idle when Healthy + no pending work (bulkhead=0 && monitored=0 && historic=0) * Healthy when Healthy + pending work + progress within threshold * Faulted when Healthy + pending work + no progress within threshold Threshold clamps to min 60 s. DemandSignal.HasPendingWork ORs the three counters. The three false-wedge cases the plan calls out all stay Healthy: idle subscription-only, slow historian backfill making progress, write-only burst with drained bulkhead. Tests (22 new, all pass): - MemoryRecycleTests (7): Tier C hard-breach requests recycle; Tier A/B hard-breach never requests; Tier C without supervisor no-ops; soft-breach at every tier never requests; None/Warming never request. - ScheduledRecycleSchedulerTests (6): ctor throws for A/B; zero/negative interval throws; tick before due no-ops; tick at/after due fires once and advances; RequestRecycleNow fires immediately without shifting schedule; multiple fires across ticks advance one interval each. - WedgeDetectorTests (9): threshold clamp to 60 s; unhealthy driver always NotApplicable; idle subscription stays Idle; pending+fresh progress stays Healthy; pending+stale progress is Faulted; MonitoredItems active but no publish is Faulted; MonitoredItems active with fresh publish stays Healthy; historian backfill with fresh progress stays Healthy; write-only burst with empty bulkhead is Idle; HasPendingWork theory for any non-zero counter. Full solution dotnet test: 989 passing (baseline 906, +83 for Phase 6.1 so far). Pre-existing Client.CLI Subscribe flake unchanged. Stream B complete. Next up: Stream C (health endpoints + structured logging). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
102 lines
4.0 KiB
C#
102 lines
4.0 KiB
C#
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Stability;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Stability;
|
|
|
|
[Trait("Category", "Unit")]
|
|
public sealed class ScheduledRecycleSchedulerTests
|
|
{
|
|
private static readonly DateTime T0 = new(2026, 4, 19, 0, 0, 0, DateTimeKind.Utc);
|
|
private static readonly TimeSpan Weekly = TimeSpan.FromDays(7);
|
|
|
|
[Theory]
|
|
[InlineData(DriverTier.A)]
|
|
[InlineData(DriverTier.B)]
|
|
public void TierAOrB_Ctor_Throws(DriverTier tier)
|
|
{
|
|
var supervisor = new FakeSupervisor();
|
|
Should.Throw<ArgumentException>(() => new ScheduledRecycleScheduler(
|
|
tier, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance));
|
|
}
|
|
|
|
[Fact]
|
|
public void ZeroOrNegativeInterval_Throws()
|
|
{
|
|
var supervisor = new FakeSupervisor();
|
|
Should.Throw<ArgumentException>(() => new ScheduledRecycleScheduler(
|
|
DriverTier.C, TimeSpan.Zero, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance));
|
|
Should.Throw<ArgumentException>(() => new ScheduledRecycleScheduler(
|
|
DriverTier.C, TimeSpan.FromSeconds(-1), T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Tick_BeforeNextRecycle_NoOp()
|
|
{
|
|
var supervisor = new FakeSupervisor();
|
|
var sch = new ScheduledRecycleScheduler(DriverTier.C, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
|
|
|
|
var fired = await sch.TickAsync(T0 + TimeSpan.FromDays(6), CancellationToken.None);
|
|
|
|
fired.ShouldBeFalse();
|
|
supervisor.RecycleCount.ShouldBe(0);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Tick_AtOrAfterNextRecycle_FiresOnce_AndAdvances()
|
|
{
|
|
var supervisor = new FakeSupervisor();
|
|
var sch = new ScheduledRecycleScheduler(DriverTier.C, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
|
|
|
|
var fired = await sch.TickAsync(T0 + Weekly + TimeSpan.FromMinutes(1), CancellationToken.None);
|
|
|
|
fired.ShouldBeTrue();
|
|
supervisor.RecycleCount.ShouldBe(1);
|
|
sch.NextRecycleUtc.ShouldBe(T0 + Weekly + Weekly);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RequestRecycleNow_Fires_Immediately_WithoutAdvancingSchedule()
|
|
{
|
|
var supervisor = new FakeSupervisor();
|
|
var sch = new ScheduledRecycleScheduler(DriverTier.C, Weekly, T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
|
|
var nextBefore = sch.NextRecycleUtc;
|
|
|
|
await sch.RequestRecycleNowAsync("memory hard-breach", CancellationToken.None);
|
|
|
|
supervisor.RecycleCount.ShouldBe(1);
|
|
supervisor.LastReason.ShouldBe("memory hard-breach");
|
|
sch.NextRecycleUtc.ShouldBe(nextBefore, "ad-hoc recycle doesn't shift the cron schedule");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task MultipleFires_AcrossTicks_AdvanceOneIntervalEach()
|
|
{
|
|
var supervisor = new FakeSupervisor();
|
|
var sch = new ScheduledRecycleScheduler(DriverTier.C, TimeSpan.FromDays(1), T0, supervisor, NullLogger<ScheduledRecycleScheduler>.Instance);
|
|
|
|
await sch.TickAsync(T0 + TimeSpan.FromDays(1) + TimeSpan.FromHours(1), CancellationToken.None);
|
|
await sch.TickAsync(T0 + TimeSpan.FromDays(2) + TimeSpan.FromHours(1), CancellationToken.None);
|
|
await sch.TickAsync(T0 + TimeSpan.FromDays(3) + TimeSpan.FromHours(1), CancellationToken.None);
|
|
|
|
supervisor.RecycleCount.ShouldBe(3);
|
|
sch.NextRecycleUtc.ShouldBe(T0 + TimeSpan.FromDays(4));
|
|
}
|
|
|
|
private sealed class FakeSupervisor : IDriverSupervisor
|
|
{
|
|
public string DriverInstanceId => "tier-c-fake";
|
|
public int RecycleCount { get; private set; }
|
|
public string? LastReason { get; private set; }
|
|
|
|
public Task RecycleAsync(string reason, CancellationToken cancellationToken)
|
|
{
|
|
RecycleCount++;
|
|
LastReason = reason;
|
|
return Task.CompletedTask;
|
|
}
|
|
}
|
|
}
|