1d9008e354
Closes out Stream B per docs/v2/implementation/phase-6-1-resilience-and-observability.md. Core.Abstractions: - IDriverSupervisor — process-level supervisor contract a Tier C driver's out-of-process topology provides (Galaxy Proxy/Supervisor implements this in a follow-up Driver.Galaxy wiring PR). Concerns: DriverInstanceId + RecycleAsync. Tier A/B drivers don't implement this; Stream B code asserts tier == C before ever calling it. Core.Stability: - MemoryRecycle — companion to MemoryTracking. On HardBreach, invokes the supervisor IFF tier == C AND a supervisor is wired. Tier A/B HardBreach logs a promotion-to-Tier-C recommendation and returns false. Soft/None/Warming never triggers a recycle at any tier. - ScheduledRecycleScheduler — Tier C opt-in periodic recycler per decision #67. Ctor throws for Tier A/B (structural guard — scheduled recycle on an in-process driver would kill every OPC UA session and every co-hosted driver). TickAsync(now) advances the schedule by one interval per fire; RequestRecycleNowAsync drives an ad-hoc recycle without shifting the cron. - WedgeDetector — demand-aware per decision #147. Classify(state, demand, now) returns: * NotApplicable when driver state != Healthy * Idle when Healthy + no pending work (bulkhead=0 && monitored=0 && historic=0) * Healthy when Healthy + pending work + progress within threshold * Faulted when Healthy + pending work + no progress within threshold Threshold clamps to min 60 s. DemandSignal.HasPendingWork ORs the three counters. The three false-wedge cases the plan calls out all stay Healthy: idle subscription-only, slow historian backfill making progress, write-only burst with drained bulkhead. Tests (22 new, all pass): - MemoryRecycleTests (7): Tier C hard-breach requests recycle; Tier A/B hard-breach never requests; Tier C without supervisor no-ops; soft-breach at every tier never requests; None/Warming never request. - ScheduledRecycleSchedulerTests (6): ctor throws for A/B; zero/negative interval throws; tick before due no-ops; tick at/after due fires once and advances; RequestRecycleNow fires immediately without shifting schedule; multiple fires across ticks advance one interval each. - WedgeDetectorTests (9): threshold clamp to 60 s; unhealthy driver always NotApplicable; idle subscription stays Idle; pending+fresh progress stays Healthy; pending+stale progress is Faulted; MonitoredItems active but no publish is Faulted; MonitoredItems active with fresh publish stays Healthy; historian backfill with fresh progress stays Healthy; write-only burst with empty bulkhead is Idle; HasPendingWork theory for any non-zero counter. Full solution dotnet test: 989 passing (baseline 906, +83 for Phase 6.1 so far). Pre-existing Client.CLI Subscribe flake unchanged. Stream B complete. Next up: Stream C (health endpoints + structured logging). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
92 lines
3.3 KiB
C#
92 lines
3.3 KiB
C#
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Stability;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Stability;
|
|
|
|
[Trait("Category", "Unit")]
|
|
public sealed class MemoryRecycleTests
|
|
{
|
|
[Fact]
|
|
public async Task TierC_HardBreach_RequestsSupervisorRecycle()
|
|
{
|
|
var supervisor = new FakeSupervisor();
|
|
var recycle = new MemoryRecycle(DriverTier.C, supervisor, NullLogger<MemoryRecycle>.Instance);
|
|
|
|
var requested = await recycle.HandleAsync(MemoryTrackingAction.HardBreach, 2_000_000_000, CancellationToken.None);
|
|
|
|
requested.ShouldBeTrue();
|
|
supervisor.RecycleCount.ShouldBe(1);
|
|
supervisor.LastReason.ShouldContain("hard-breach");
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(DriverTier.A)]
|
|
[InlineData(DriverTier.B)]
|
|
public async Task InProcessTier_HardBreach_NeverRequestsRecycle(DriverTier tier)
|
|
{
|
|
var supervisor = new FakeSupervisor();
|
|
var recycle = new MemoryRecycle(tier, supervisor, NullLogger<MemoryRecycle>.Instance);
|
|
|
|
var requested = await recycle.HandleAsync(MemoryTrackingAction.HardBreach, 2_000_000_000, CancellationToken.None);
|
|
|
|
requested.ShouldBeFalse("Tier A/B hard-breach logs a promotion recommendation only (decisions #74, #145)");
|
|
supervisor.RecycleCount.ShouldBe(0);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task TierC_WithoutSupervisor_HardBreach_NoOp()
|
|
{
|
|
var recycle = new MemoryRecycle(DriverTier.C, supervisor: null, NullLogger<MemoryRecycle>.Instance);
|
|
|
|
var requested = await recycle.HandleAsync(MemoryTrackingAction.HardBreach, 2_000_000_000, CancellationToken.None);
|
|
|
|
requested.ShouldBeFalse("no supervisor → no recycle path; action logged only");
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(DriverTier.A)]
|
|
[InlineData(DriverTier.B)]
|
|
[InlineData(DriverTier.C)]
|
|
public async Task SoftBreach_NeverRequestsRecycle(DriverTier tier)
|
|
{
|
|
var supervisor = new FakeSupervisor();
|
|
var recycle = new MemoryRecycle(tier, supervisor, NullLogger<MemoryRecycle>.Instance);
|
|
|
|
var requested = await recycle.HandleAsync(MemoryTrackingAction.SoftBreach, 1_000_000_000, CancellationToken.None);
|
|
|
|
requested.ShouldBeFalse("soft-breach is surface-only at every tier");
|
|
supervisor.RecycleCount.ShouldBe(0);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(MemoryTrackingAction.None)]
|
|
[InlineData(MemoryTrackingAction.Warming)]
|
|
public async Task NonBreachActions_NoOp(MemoryTrackingAction action)
|
|
{
|
|
var supervisor = new FakeSupervisor();
|
|
var recycle = new MemoryRecycle(DriverTier.C, supervisor, NullLogger<MemoryRecycle>.Instance);
|
|
|
|
var requested = await recycle.HandleAsync(action, 100_000_000, CancellationToken.None);
|
|
|
|
requested.ShouldBeFalse();
|
|
supervisor.RecycleCount.ShouldBe(0);
|
|
}
|
|
|
|
private sealed class FakeSupervisor : IDriverSupervisor
|
|
{
|
|
public string DriverInstanceId => "fake-tier-c";
|
|
public int RecycleCount { get; private set; }
|
|
public string? LastReason { get; private set; }
|
|
|
|
public Task RecycleAsync(string reason, CancellationToken cancellationToken)
|
|
{
|
|
RecycleCount++;
|
|
LastReason = reason;
|
|
return Task.CompletedTask;
|
|
}
|
|
}
|
|
}
|