Compare commits

...

2 Commits

Author SHA1 Message Date
Joseph Doherty
5033609944 FOCAS Tier-C PR D — supervisor + backoff + crash-loop breaker + heartbeat monitor. Fourth of 5 PRs for #220. Ships the resilience harness that sits between the driver's IFocasClient requests and the Tier-C Host process, so a crashing Fwlib32.dll takes down only the Host (not the main server), gets respawned on a backoff ladder, and opens a circuit with a sticky operator alert when the crash rate is pathological. Same shape as Galaxy Tier-C so the Admin /hosts surface has a single mental model. New Supervisor/ namespace in Driver.FOCAS (.NET 10, Proxy-side): Backoff with the 5s→15s→60s default ladder + StableRunThreshold that resets the index after a 2-min clean run (so a one-off crash after hours of steady-state doesn't restart from the top); CircuitBreaker with 3-crashes-in-5-min threshold + escalating 1h→4h→manual-reset cooldown ladder + StickyAlertActive flag that persists across cooldowns until AcknowledgeAndReset is called; HeartbeatMonitor tracking ConsecutiveMisses against the 3-misses-kill threshold + LastAckUtc for telemetry; IHostProcessLauncher abstraction over "spawn Host process + produce an IFocasClient connected to it" so the supervisor stays I/O-free and fully testable with a fake launcher that can be told to throw on specific attempts (production wiring over Process.Start + FocasIpcClient.ConnectAsync is the PR E ops-glue concern); FocasHostSupervisor orchestrating them — GetOrLaunchAsync cycles through backoff until either a client is returned or the breaker opens (surfaced as InvalidOperationException so the driver maps to BadDeviceFailure), NotifyHostDeadAsync fans out the unavailable event + terminates the current launcher + records the crash without blocking (so heartbeat-loss detection can short-circuit subscriber fan-out and let the next GetOrLaunchAsync handle the respawn), AcknowledgeAndReset is the operator-clear path, OnUnavailable event for Admin /hosts wiring + ObservedCrashes + BackoffAttempt + StickyAlertActive for telemetry. 14 new unit tests across SupervisorTests.cs: Backoff (default sequence, clamping, RecordStableRun resets), CircuitBreaker (below threshold allowed, opens at threshold, escalates cooldown after second open, ManualReset clears state), HeartbeatMonitor (3 consecutive misses declares dead, ack resets counter), FocasHostSupervisor (first-launch success, retry-with-backoff after transient failure, repeated failures open breaker + surface InvalidOperationException, NotifyHostDeadAsync terminates + fan-outs + increments crash count, AcknowledgeAndReset clears sticky, Dispose terminates). Full FOCAS driver tests now 186/186 green (172 + 14 new). No changes to IFocasClient DI contract; existing FakeFocasClient-based tests unaffected. PR E wires the real Process-based IHostProcessLauncher + NSSM install scripts + MMF post-mortem + docs.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-20 14:17:23 -04:00
9034294b77 Merge pull request 'FOCAS Tier-C PR C — IPC path end-to-end' (#171) from focas-tier-c-pr-c-ipc-proxy into v2 2026-04-20 14:13:33 -04:00
6 changed files with 568 additions and 0 deletions

View File

@@ -0,0 +1,30 @@
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Supervisor;
/// <summary>
/// Respawn-with-backoff schedule for the FOCAS Host process. Matches Galaxy Tier-C:
/// 5s → 15s → 60s cap. A sustained stable run (default 2 min) resets the index so a
/// one-off crash after hours of steady-state doesn't start from the top of the ladder.
/// </summary>
public sealed class Backoff
{
public static TimeSpan[] DefaultSequence { get; } =
[TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(60)];
public TimeSpan StableRunThreshold { get; init; } = TimeSpan.FromMinutes(2);
private readonly TimeSpan[] _sequence;
private int _index;
public Backoff(TimeSpan[]? sequence = null) => _sequence = sequence ?? DefaultSequence;
public TimeSpan Next()
{
var delay = _sequence[Math.Min(_index, _sequence.Length - 1)];
_index++;
return delay;
}
public void RecordStableRun() => _index = 0;
public int AttemptIndex => _index;
}

View File

@@ -0,0 +1,69 @@
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Supervisor;
/// <summary>
/// Crash-loop circuit breaker for the FOCAS Host. Matches Galaxy Tier-C defaults:
/// 3 crashes within 5 minutes opens the breaker; cooldown escalates 1h → 4h → manual
/// reset. A sticky alert stays live until the operator explicitly clears it so
/// recurring crashes can't silently burn through the cooldown ladder overnight.
/// </summary>
public sealed class CircuitBreaker
{
public int CrashesAllowedPerWindow { get; init; } = 3;
public TimeSpan Window { get; init; } = TimeSpan.FromMinutes(5);
public TimeSpan[] CooldownEscalation { get; init; } =
[TimeSpan.FromHours(1), TimeSpan.FromHours(4), TimeSpan.MaxValue];
private readonly List<DateTime> _crashesUtc = [];
private DateTime? _openSinceUtc;
private int _escalationLevel;
public bool StickyAlertActive { get; private set; }
/// <summary>
/// Records a crash + returns <c>true</c> if the supervisor may respawn. On
/// <c>false</c>, <paramref name="cooldownRemaining"/> is how long to wait before
/// trying again (<c>TimeSpan.MaxValue</c> means manual reset required).
/// </summary>
public bool TryRecordCrash(DateTime utcNow, out TimeSpan cooldownRemaining)
{
if (_openSinceUtc is { } openedAt)
{
var cooldown = CooldownEscalation[Math.Min(_escalationLevel, CooldownEscalation.Length - 1)];
if (cooldown == TimeSpan.MaxValue)
{
cooldownRemaining = TimeSpan.MaxValue;
return false;
}
if (utcNow - openedAt < cooldown)
{
cooldownRemaining = cooldown - (utcNow - openedAt);
return false;
}
_openSinceUtc = null;
_escalationLevel++;
}
_crashesUtc.RemoveAll(t => utcNow - t > Window);
_crashesUtc.Add(utcNow);
if (_crashesUtc.Count > CrashesAllowedPerWindow)
{
_openSinceUtc = utcNow;
StickyAlertActive = true;
cooldownRemaining = CooldownEscalation[Math.Min(_escalationLevel, CooldownEscalation.Length - 1)];
return false;
}
cooldownRemaining = TimeSpan.Zero;
return true;
}
public void ManualReset()
{
_crashesUtc.Clear();
_openSinceUtc = null;
_escalationLevel = 0;
StickyAlertActive = false;
}
}

View File

@@ -0,0 +1,159 @@
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Supervisor;
/// <summary>
/// Ties <see cref="IHostProcessLauncher"/> + <see cref="Backoff"/> +
/// <see cref="CircuitBreaker"/> + <see cref="HeartbeatMonitor"/> into one object the
/// driver asks for <c>IFocasClient</c>s. On a detected crash (process exit or
/// heartbeat loss) the supervisor fans out <c>BadCommunicationError</c> to all
/// subscribers via the <see cref="OnUnavailable"/> callback, then respawns with
/// backoff unless the breaker is open.
/// </summary>
/// <remarks>
/// The supervisor itself is I/O-free — it doesn't know how to spawn processes, probe
/// pipes, or send heartbeats. Production wires the concrete
/// <see cref="IHostProcessLauncher"/> over <c>FocasIpcClient</c> + <c>Process</c>;
/// tests drive the same state machine with a deterministic launcher stub.
/// </remarks>
public sealed class FocasHostSupervisor : IDisposable
{
private readonly IHostProcessLauncher _launcher;
private readonly Backoff _backoff;
private readonly CircuitBreaker _breaker;
private readonly Func<DateTime> _clock;
private IFocasClient? _current;
private DateTime _currentStartedUtc;
private bool _disposed;
public FocasHostSupervisor(
IHostProcessLauncher launcher,
Backoff? backoff = null,
CircuitBreaker? breaker = null,
Func<DateTime>? clock = null)
{
_launcher = launcher ?? throw new ArgumentNullException(nameof(launcher));
_backoff = backoff ?? new Backoff();
_breaker = breaker ?? new CircuitBreaker();
_clock = clock ?? (() => DateTime.UtcNow);
}
/// <summary>Raised with a short reason string whenever the Host goes unavailable (crash / heartbeat loss / breaker-open).</summary>
public event Action<string>? OnUnavailable;
/// <summary>Crash count observed in the current process lifetime. Exposed for /hosts Admin telemetry.</summary>
public int ObservedCrashes { get; private set; }
/// <summary><c>true</c> if the crash-loop breaker has latched a sticky alert that needs operator reset.</summary>
public bool StickyAlertActive => _breaker.StickyAlertActive;
public int BackoffAttempt => _backoff.AttemptIndex;
/// <summary>
/// Returns the current live client. If none, tries to launch — applying the
/// backoff schedule between attempts and stopping once the breaker opens.
/// </summary>
public async Task<IFocasClient> GetOrLaunchAsync(CancellationToken ct)
{
ThrowIfDisposed();
if (_current is not null && _launcher.IsProcessAlive) return _current;
return await LaunchWithBackoffAsync(ct).ConfigureAwait(false);
}
/// <summary>
/// Called by the heartbeat task each time a miss threshold is crossed.
/// Treated as a crash: fan out Bad status + attempt respawn.
/// </summary>
public async Task NotifyHostDeadAsync(string reason, CancellationToken ct)
{
ThrowIfDisposed();
OnUnavailable?.Invoke(reason);
ObservedCrashes++;
try { await _launcher.TerminateAsync(ct).ConfigureAwait(false); }
catch { /* best effort */ }
_current?.Dispose();
_current = null;
if (!_breaker.TryRecordCrash(_clock(), out var cooldown))
{
OnUnavailable?.Invoke(cooldown == TimeSpan.MaxValue
? "circuit-breaker-open-manual-reset-required"
: $"circuit-breaker-open-cooldown-{cooldown:g}");
return;
}
// Successful crash recording — do not respawn synchronously; GetOrLaunchAsync will
// pick up the attempt on the next call. Keeps the fan-out fast.
}
/// <summary>Operator action — clear the sticky alert + reset the breaker.</summary>
public void AcknowledgeAndReset()
{
_breaker.ManualReset();
_backoff.RecordStableRun();
}
private async Task<IFocasClient> LaunchWithBackoffAsync(CancellationToken ct)
{
while (true)
{
if (_breaker.StickyAlertActive)
{
if (!_breaker.TryRecordCrash(_clock(), out var cooldown) && cooldown == TimeSpan.MaxValue)
throw new InvalidOperationException(
"FOCAS Host circuit breaker is open and awaiting manual reset. " +
"See Admin /hosts; call AcknowledgeAndReset after investigating the Host log.");
}
try
{
_current = await _launcher.LaunchAsync(ct).ConfigureAwait(false);
_currentStartedUtc = _clock();
// If the launch sequence itself takes long enough to count as a stable run,
// reset the backoff ladder immediately.
if (_clock() - _currentStartedUtc >= _backoff.StableRunThreshold)
_backoff.RecordStableRun();
return _current;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
OnUnavailable?.Invoke($"launch-failed: {ex.Message}");
ObservedCrashes++;
if (!_breaker.TryRecordCrash(_clock(), out var cooldown))
{
var hint = cooldown == TimeSpan.MaxValue
? "manual reset required"
: $"cooldown {cooldown:g}";
throw new InvalidOperationException(
$"FOCAS Host circuit breaker opened after {ObservedCrashes} crashes — {hint}.", ex);
}
var delay = _backoff.Next();
await Task.Delay(delay, ct).ConfigureAwait(false);
}
}
}
/// <summary>Called from the heartbeat loop after a successful ack run — relaxes the backoff ladder.</summary>
public void NotifyStableRun()
{
if (_current is null) return;
if (_clock() - _currentStartedUtc >= _backoff.StableRunThreshold)
_backoff.RecordStableRun();
}
public void Dispose()
{
if (_disposed) return;
_disposed = true;
try { _launcher.TerminateAsync(CancellationToken.None).GetAwaiter().GetResult(); }
catch { /* best effort */ }
_current?.Dispose();
_current = null;
}
private void ThrowIfDisposed()
{
if (_disposed) throw new ObjectDisposedException(nameof(FocasHostSupervisor));
}
}

View File

@@ -0,0 +1,29 @@
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Supervisor;
/// <summary>
/// Tracks missed heartbeats from the FOCAS Host. 2s cadence + 3 consecutive misses =
/// host declared dead (~6s detection). Same defaults as Galaxy Tier-C so operators
/// see the same cadence across hosts on the /hosts Admin page.
/// </summary>
public sealed class HeartbeatMonitor
{
public int MissesUntilDead { get; init; } = 3;
public TimeSpan Cadence { get; init; } = TimeSpan.FromSeconds(2);
public int ConsecutiveMisses { get; private set; }
public DateTime? LastAckUtc { get; private set; }
public void RecordAck(DateTime utcNow)
{
ConsecutiveMisses = 0;
LastAckUtc = utcNow;
}
/// <summary>Records a missed heartbeat; returns <c>true</c> when the death threshold is crossed.</summary>
public bool RecordMiss()
{
ConsecutiveMisses++;
return ConsecutiveMisses >= MissesUntilDead;
}
}

View File

@@ -0,0 +1,32 @@
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Supervisor;
/// <summary>
/// Abstraction over the act of spawning a FOCAS Host process and obtaining an
/// <see cref="IFocasClient"/> connected to it. Production wires this to a real
/// <c>Process.Start</c> + <c>FocasIpcClient.ConnectAsync</c>; tests use a fake that
/// exposes deterministic failure modes so the supervisor logic can be stressed
/// without spawning actual exes.
/// </summary>
public interface IHostProcessLauncher
{
/// <summary>
/// Spawn a new Host process (if one isn't already running) and return a live
/// client session. Throws on unrecoverable errors; transient errors (e.g. Host
/// not ready yet) should throw <see cref="TimeoutException"/> so the supervisor
/// applies the backoff ladder.
/// </summary>
Task<IFocasClient> LaunchAsync(CancellationToken ct);
/// <summary>
/// Terminate the Host process if one is running. Called on Dispose and after a
/// heartbeat loss is detected.
/// </summary>
Task TerminateAsync(CancellationToken ct);
/// <summary>
/// <c>true</c> when the most recently spawned Host process is still alive.
/// Supervisor polls this at heartbeat cadence; going <c>false</c> without a
/// clean shutdown counts as a crash.
/// </summary>
bool IsProcessAlive { get; }
}

View File

@@ -0,0 +1,249 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Supervisor;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
[Trait("Category", "Unit")]
public sealed class BackoffTests
{
[Fact]
public void Default_sequence_is_5s_15s_60s_then_clamped()
{
var b = new Backoff();
b.Next().ShouldBe(TimeSpan.FromSeconds(5));
b.Next().ShouldBe(TimeSpan.FromSeconds(15));
b.Next().ShouldBe(TimeSpan.FromSeconds(60));
b.Next().ShouldBe(TimeSpan.FromSeconds(60));
b.Next().ShouldBe(TimeSpan.FromSeconds(60));
}
[Fact]
public void RecordStableRun_resets_the_ladder_to_the_start()
{
var b = new Backoff();
b.Next(); b.Next();
b.AttemptIndex.ShouldBe(2);
b.RecordStableRun();
b.AttemptIndex.ShouldBe(0);
b.Next().ShouldBe(TimeSpan.FromSeconds(5));
}
}
[Trait("Category", "Unit")]
public sealed class CircuitBreakerTests
{
[Fact]
public void Allows_crashes_below_threshold()
{
var b = new CircuitBreaker();
var now = DateTime.UtcNow;
b.TryRecordCrash(now, out _).ShouldBeTrue();
b.TryRecordCrash(now.AddSeconds(1), out _).ShouldBeTrue();
b.TryRecordCrash(now.AddSeconds(2), out _).ShouldBeTrue();
b.StickyAlertActive.ShouldBeFalse();
}
[Fact]
public void Opens_when_exceeding_threshold_in_window()
{
var b = new CircuitBreaker();
var now = DateTime.UtcNow;
b.TryRecordCrash(now, out _);
b.TryRecordCrash(now.AddSeconds(1), out _);
b.TryRecordCrash(now.AddSeconds(2), out _);
b.TryRecordCrash(now.AddSeconds(3), out var cooldown).ShouldBeFalse();
cooldown.ShouldBe(TimeSpan.FromHours(1));
b.StickyAlertActive.ShouldBeTrue();
}
[Fact]
public void Escalates_cooldown_after_second_open()
{
var b = new CircuitBreaker();
var t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
// First burst — 4 crashes opens breaker with 1h cooldown.
for (var i = 0; i < 4; i++) b.TryRecordCrash(t0.AddSeconds(i), out _);
b.StickyAlertActive.ShouldBeTrue();
// Wait past cooldown. The first crash after cooldown-elapsed resets _openSinceUtc and
// bumps escalation level; the next 3 crashes then re-open with the escalated 4h cooldown.
b.TryRecordCrash(t0.AddHours(1).AddMinutes(1), out _);
var t1 = t0.AddHours(1).AddMinutes(1).AddSeconds(1);
b.TryRecordCrash(t1, out _);
b.TryRecordCrash(t1.AddSeconds(1), out _);
b.TryRecordCrash(t1.AddSeconds(2), out var cooldown).ShouldBeFalse();
cooldown.ShouldBe(TimeSpan.FromHours(4));
}
[Fact]
public void ManualReset_clears_everything()
{
var b = new CircuitBreaker();
var now = DateTime.UtcNow;
for (var i = 0; i < 5; i++) b.TryRecordCrash(now.AddSeconds(i), out _);
b.StickyAlertActive.ShouldBeTrue();
b.ManualReset();
b.StickyAlertActive.ShouldBeFalse();
b.TryRecordCrash(now.AddSeconds(10), out _).ShouldBeTrue();
}
}
[Trait("Category", "Unit")]
public sealed class HeartbeatMonitorTests
{
[Fact]
public void Three_consecutive_misses_declares_dead()
{
var m = new HeartbeatMonitor();
m.RecordMiss().ShouldBeFalse();
m.RecordMiss().ShouldBeFalse();
m.RecordMiss().ShouldBeTrue();
}
[Fact]
public void Ack_resets_the_miss_counter()
{
var m = new HeartbeatMonitor();
m.RecordMiss(); m.RecordMiss();
m.ConsecutiveMisses.ShouldBe(2);
m.RecordAck(DateTime.UtcNow);
m.ConsecutiveMisses.ShouldBe(0);
}
}
[Trait("Category", "Unit")]
public sealed class FocasHostSupervisorTests
{
private sealed class FakeLauncher : IHostProcessLauncher
{
public int LaunchAttempts { get; private set; }
public int Terminations { get; private set; }
public Queue<Func<IFocasClient>> Plan { get; } = new();
public bool IsProcessAlive { get; set; }
public Task<IFocasClient> LaunchAsync(CancellationToken ct)
{
LaunchAttempts++;
if (Plan.Count == 0) throw new InvalidOperationException("FakeLauncher plan exhausted");
var next = Plan.Dequeue()();
IsProcessAlive = true;
return Task.FromResult(next);
}
public Task TerminateAsync(CancellationToken ct)
{
Terminations++;
IsProcessAlive = false;
return Task.CompletedTask;
}
}
private sealed class StubFocasClient : IFocasClient
{
public bool IsConnected => true;
public Task ConnectAsync(FocasHostAddress address, TimeSpan timeout, CancellationToken ct) => Task.CompletedTask;
public Task<(object? value, uint status)> ReadAsync(FocasAddress a, FocasDataType t, CancellationToken ct) =>
Task.FromResult<(object?, uint)>((0, 0));
public Task<uint> WriteAsync(FocasAddress a, FocasDataType t, object? v, CancellationToken ct) => Task.FromResult(0u);
public Task<bool> ProbeAsync(CancellationToken ct) => Task.FromResult(true);
public void Dispose() { }
}
[Fact]
public async Task GetOrLaunch_returns_client_on_first_success()
{
var launcher = new FakeLauncher();
launcher.Plan.Enqueue(() => new StubFocasClient());
var supervisor = new FocasHostSupervisor(launcher);
var client = await supervisor.GetOrLaunchAsync(TestContext.Current.CancellationToken);
client.ShouldNotBeNull();
launcher.LaunchAttempts.ShouldBe(1);
}
[Fact]
public async Task GetOrLaunch_retries_after_transient_failure_with_backoff()
{
var launcher = new FakeLauncher();
launcher.Plan.Enqueue(() => throw new TimeoutException("pipe not ready"));
launcher.Plan.Enqueue(() => new StubFocasClient());
var backoff = new Backoff([TimeSpan.FromMilliseconds(10), TimeSpan.FromMilliseconds(20)]);
var supervisor = new FocasHostSupervisor(launcher, backoff);
var unavailableMessages = new List<string>();
supervisor.OnUnavailable += m => unavailableMessages.Add(m);
var client = await supervisor.GetOrLaunchAsync(TestContext.Current.CancellationToken);
client.ShouldNotBeNull();
launcher.LaunchAttempts.ShouldBe(2);
unavailableMessages.Count.ShouldBe(1);
unavailableMessages[0].ShouldContain("launch-failed");
}
[Fact]
public async Task Repeated_launch_failures_open_breaker_and_surface_InvalidOperation()
{
var launcher = new FakeLauncher();
for (var i = 0; i < 10; i++)
launcher.Plan.Enqueue(() => throw new InvalidOperationException("simulated host refused"));
var supervisor = new FocasHostSupervisor(
launcher,
backoff: new Backoff([TimeSpan.FromMilliseconds(1)]),
breaker: new CircuitBreaker { CrashesAllowedPerWindow = 2, Window = TimeSpan.FromMinutes(5) });
var ex = await Should.ThrowAsync<InvalidOperationException>(async () =>
await supervisor.GetOrLaunchAsync(TestContext.Current.CancellationToken));
ex.Message.ShouldContain("circuit breaker");
supervisor.StickyAlertActive.ShouldBeTrue();
}
[Fact]
public async Task NotifyHostDeadAsync_terminates_current_and_fans_out_unavailable()
{
var launcher = new FakeLauncher();
launcher.Plan.Enqueue(() => new StubFocasClient());
var supervisor = new FocasHostSupervisor(launcher);
var messages = new List<string>();
supervisor.OnUnavailable += m => messages.Add(m);
await supervisor.GetOrLaunchAsync(TestContext.Current.CancellationToken);
await supervisor.NotifyHostDeadAsync("heartbeat-loss", TestContext.Current.CancellationToken);
launcher.Terminations.ShouldBe(1);
messages.ShouldContain("heartbeat-loss");
supervisor.ObservedCrashes.ShouldBe(1);
}
[Fact]
public async Task AcknowledgeAndReset_clears_sticky_alert()
{
var launcher = new FakeLauncher();
for (var i = 0; i < 10; i++)
launcher.Plan.Enqueue(() => throw new InvalidOperationException("refused"));
var supervisor = new FocasHostSupervisor(
launcher,
backoff: new Backoff([TimeSpan.FromMilliseconds(1)]),
breaker: new CircuitBreaker { CrashesAllowedPerWindow = 1 });
try { await supervisor.GetOrLaunchAsync(TestContext.Current.CancellationToken); } catch { }
supervisor.StickyAlertActive.ShouldBeTrue();
supervisor.AcknowledgeAndReset();
supervisor.StickyAlertActive.ShouldBeFalse();
}
[Fact]
public async Task Dispose_terminates_host_process()
{
var launcher = new FakeLauncher();
launcher.Plan.Enqueue(() => new StubFocasClient());
var supervisor = new FocasHostSupervisor(launcher);
await supervisor.GetOrLaunchAsync(TestContext.Current.CancellationToken);
supervisor.Dispose();
launcher.Terminations.ShouldBe(1);
}
}