dohertj2/lmxopcua
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge fix/827-worker-deadline

Browse Source
This commit is contained in:
Joseph Doherty
2026-11-11 02:34:47 -05:00
parent 5bfe8386e6 5962cc3d41
commit 9c0cdc05ef
15 changed files with 1031 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
+20 -4
View File
@@ -491,6 +491,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// (<see cref="DriverInstanceActor.WriteAttribute.IssuedAt"/>) means the same thing on both sides.</summary>
private readonly TimeProvider _writeClock;
/// <summary>
/// Test seam: the PostStop shutdown budget every real (non-stub) child is spawned with; null =
/// <see cref=DriverInstanceActor.DefaultShutdownTimeout/>. A test that parks a stopped child in its
/// PostStop to observe the retired-but-not-terminated window raises it, so the window cannot close under
/// load before the test has looked (lmxopcua#828).
/// </summary>
private readonly TimeSpan? _childShutdownTimeout;
/// <summary>Whether any redundancy snapshot has reached this host — the HistoryUpdate gate stays
/// closed until one has (see <see cref="PrimaryGatePolicy.ShouldServeHistoryUpdate"/>).</summary>
private bool _redundancySnapshotReceived;
@@ -738,6 +746,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// Ask is derived (<see cref="DriverWriteBounds.HostAsk"/>). Null ⇒ <see cref="DriverWriteBounds.Default"/>.</param>
/// <param name="writeClock">The clock the write bounds (and the host's write Ask) run on, shared with every
/// child. Null ⇒ <see cref="TimeProvider.System"/>.</param>
/// <param name="childShutdownTimeout">Test seam: each real child's PostStop shutdown budget; null ⇒
/// <see cref="DriverInstanceActor.DefaultShutdownTimeout"/> (lmxopcua#828).</param>
/// <returns>The Akka.NET <see cref="Akka.Actor.Props"/> used to spawn this actor.</returns>
public static Props Props(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
@@ -776,7 +786,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
NativeAlarmAckPreflightView? ackPreflightView = null,
DriverWriteBounds? writeBounds = null,
TimeProvider? writeClock = null,
IScriptRunner? scriptedAlarmRunner = null) =>
IScriptRunner? scriptedAlarmRunner = null,
TimeSpan? childShutdownTimeout = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -790,7 +801,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView,
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner));
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -885,6 +896,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// Ask is derived (<see cref="DriverWriteBounds.HostAsk"/>). Null ⇒ <see cref="DriverWriteBounds.Default"/>.</param>
/// <param name="writeClock">The clock the write bounds (and the host's write Ask) run on, shared with every
/// child. Null ⇒ <see cref="TimeProvider.System"/>.</param>
/// <param name="childShutdownTimeout">Test seam: each real child's PostStop shutdown budget; null ⇒
/// <see cref="DriverInstanceActor.DefaultShutdownTimeout"/> (lmxopcua#828).</param>
public DriverHostActor(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
CommonsNodeId localNode,
@@ -922,8 +935,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
NativeAlarmAckPreflightView? ackPreflightView = null,
DriverWriteBounds? writeBounds = null,
TimeProvider? writeClock = null,
IScriptRunner? scriptedAlarmRunner = null)
IScriptRunner? scriptedAlarmRunner = null,
TimeSpan? childShutdownTimeout = null)
{
_childShutdownTimeout = childShutdownTimeout;
_scriptedAlarmRunner = scriptedAlarmRunner;
_writeBounds = writeBounds ?? DriverWriteBounds.Default;
_writeClock = writeClock ?? TimeProvider.System;
@@ -3957,7 +3972,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
clusterId: clusterId,
invoker: invoker,
writeTimeout: _writeBounds.Timeout,
writeClock: _writeClock),
writeClock: _writeClock,
shutdownTimeout: _childShutdownTimeout),
actorName);
child.Tell(new DriverInstanceActor.InitializeRequested(spec.DriverConfig));
}
src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs
+100 -7
View File
@@ -29,6 +29,9 @@ internal sealed class WorkerHost
/// <summary>The watchdog fails fast this long after the host's own hard deadline, so the host normally wins.</summary>
private static readonly TimeSpan WatchdogMargin = TimeSpan.FromMilliseconds(500);
/// <summary>The watchdog's longest sleep between passes.</summary>
private static readonly TimeSpan WatchdogPoll = TimeSpan.FromMilliseconds(50);
private readonly NamedPipeClientStream _pipe;
private readonly string _nonce;
private readonly int _parentPid;
@@ -75,6 +78,19 @@ internal sealed class WorkerHost
private TimeSpan _hardGrace = TimeSpan.FromSeconds(1);
private TimeSpan _heartbeatInterval = TimeSpan.FromSeconds(1);
private ScriptPeCacheStore? _peCache;
/// <summary>
/// lmxopcua#827: this worker runs each evaluation's soft clock from the run's start and reports the crossing
/// (<c>EvalResults.soft_expired_eval_ids</c>); the host asked for it in <c>Hello</c> and we echoed it in
/// <c>Ready</c>. Off (an older host): the host's own clock, started at submit, is the soft deadline.
/// </summary>
private bool _workerSoftClock;
/// <summary>Wakes the watchdog early when a run's soft deadline is sooner than its next planned pass.</summary>
private readonly AutoResetEvent _watchdogWake = new(false);
/// <summary><see cref="Stopwatch.GetTimestamp"/> of the watchdog's next planned pass.</summary>
private long _watchdogNextPass;
private long _seq;
public WorkerHost(NamedPipeClientStream pipe, string nonce, int parentPid)
@@ -109,6 +125,7 @@ internal sealed class WorkerHost
Nonce = _nonce,
Pid = Environment.ProcessId,
RuntimeVersion = Environment.Version.ToString(),
WorkerSoftClock = _workerSoftClock,
},
});
if (ScriptWorkerFrameCodec.UseBlockingIo) ScriptWorkerFrameCodec.Write(_pipe, ready, _maxFrameBytes);
@@ -140,6 +157,7 @@ internal sealed class WorkerHost
if (hello.HardGraceMs > 0) _hardGrace = TimeSpan.FromMilliseconds(hello.HardGraceMs);
if (hello.HeartbeatIntervalMs > 0) _heartbeatInterval = TimeSpan.FromMilliseconds(hello.HeartbeatIntervalMs);
if (hello.CompileThreads > 0) _compileThreads = (int)Math.Min(hello.CompileThreads, 64);
_workerSoftClock = hello.WorkerSoftClock;
if (!string.IsNullOrEmpty(hello.PeCacheDir) && hello.PeCacheKey.Length == ScriptPeCacheStore.KeyLength)
{
// Design §7: the host owns the key; it reaches us only here, never through the environment or a file.
@@ -425,11 +443,17 @@ internal sealed class WorkerHost
var logger = _scriptLogger.ForContext(ForwardingLogSink.KeyProperty, request.Key);
var soft = TimeSpan.FromMilliseconds(Math.Max(1, request.SoftTimeoutMs));
// Cancelled by the host's Cancel frame at the soft deadline, never by our own clock (see the proto).
// Cancelled by the host's Cancel frame at the soft deadline, never by our own clock (see the proto): with the
// worker soft clock (lmxopcua#827) the watchdog only REPORTS the crossing; the host answers its caller, then
// cancels.
using var cts = new CancellationTokenSource();
var grace = request.HardGraceMs > 0 ? TimeSpan.FromMilliseconds(request.HardGraceMs) : _hardGrace;
var running = new RunningEval(request.Key, Stopwatch.GetTimestamp(), soft + grace + WatchdogMargin, cts);
var started = Stopwatch.GetTimestamp();
var running = new RunningEval(request.EvalId, request.Key, started, soft, grace, cts);
_running[(long)request.EvalId] = running;
// The watchdog plans its next pass at most WatchdogPoll ahead; a budget shorter than that wakes it now, so the
// soft notice is not late by up to a poll interval.
if (_workerSoftClock && running.SoftDue < Interlocked.Read(ref _watchdogNextPass)) _watchdogWake.Set();
var before = GC.GetAllocatedBytesForCurrentThread();
var sw = Stopwatch.StartNew();
try
@@ -465,22 +489,68 @@ internal sealed class WorkerHost
return result;
}
/// <summary>
/// The watchdog: reports soft-deadline crossings (worker soft clock, lmxopcua#827) and fails the process fast for
/// a run past its hard deadline that the host has not killed.
/// </summary>
/// <remarks>
/// The hard deadline it enforces is the host's plus <see cref="WatchdogMargin"/>, from the same anchor the host
/// uses: the run's start (T + G) on the host clock, or — with the worker soft clock — the soft notice it sent
/// (G), which is where the host arms its kill. So the host normally wins and the exit names the culprit.
/// </remarks>
private void WatchdogLoop()
{
while (true)
{
Thread.Sleep(50);
var now = Stopwatch.GetTimestamp();
var nextPass = now + Ticks(WatchdogPoll);
List<ulong>? expired = null;
foreach (var (_, run) in _running)
{
if (Stopwatch.GetElapsedTime(run.StartedTimestamp) > run.FailFastAfter)
if (_workerSoftClock)
{
// The host should have killed us already; this names the culprit in the exit/dump.
Environment.FailFast($"ScriptWorker: script '{run.Key}' exceeded its hard deadline ({run.FailFastAfter.TotalMilliseconds:F0} ms)");
if (run.SoftReportedAt == 0)
{
if (now >= run.SoftDue)
{
run.SoftReportedAt = now;
(expired ??= []).Add(run.EvalId);
}
else if (run.SoftDue < nextPass)
{
nextPass = run.SoftDue;
}
}
else if (now - run.SoftReportedAt > Ticks(run.Grace + WatchdogMargin))
{
FailFast(run, run.Grace + WatchdogMargin, "its soft deadline");
}
}
else if (now - run.StartedTimestamp > Ticks(run.Soft + run.Grace + WatchdogMargin))
{
FailFast(run, run.Soft + run.Grace + WatchdogMargin, "its start");
}
}
if (expired is not null)
{
var notice = new EvalResults();
notice.SoftExpiredEvalIds.AddRange(expired);
Send(new Envelope { Results = notice });
}
Interlocked.Exchange(ref _watchdogNextPass, nextPass);
var wait = Stopwatch.GetElapsedTime(Stopwatch.GetTimestamp(), nextPass);
if (wait > TimeSpan.Zero) _watchdogWake.WaitOne(wait);
}
static void FailFast(RunningEval run, TimeSpan after, string anchor)
{
// The host should have killed us already; this names the culprit in the exit/dump.
Environment.FailFast($"ScriptWorker: script '{run.Key}' exceeded its hard deadline ({after.TotalMilliseconds:F0} ms after {anchor})");
}
}
private static long Ticks(TimeSpan span) => (long)(span.TotalSeconds * Stopwatch.Frequency);
private async Task HeartbeatLoopAsync()
{
using var timer = new PeriodicTimer(_heartbeatInterval);
@@ -609,7 +679,30 @@ internal sealed class WorkerHost
Environment.Exit(ScriptWorkerEnvironment.ExitForbiddenAssembly);
}
private sealed record RunningEval(string Key, long StartedTimestamp, TimeSpan FailFastAfter, CancellationTokenSource Cancellation);
/// <summary>One evaluation on an evaluation thread, as the watchdog sees it.</summary>
private sealed class RunningEval(ulong evalId, string key, long startedTimestamp, TimeSpan soft, TimeSpan grace,
CancellationTokenSource cancellation)
{
public ulong EvalId { get; } = evalId;
public string Key { get; } = key;
/// <summary><see cref="Stopwatch.GetTimestamp"/> when the evaluation thread started the run.</summary>
public long StartedTimestamp { get; } = startedTimestamp;
/// <summary>T.</summary>
public TimeSpan Soft { get; } = soft;
/// <summary>The run's start + T.</summary>
public long SoftDue { get; } = startedTimestamp + Ticks(soft);
public TimeSpan Grace { get; } = grace;
public CancellationTokenSource Cancellation { get; } = cancellation;
/// <summary>When the watchdog reported the soft crossing to the host; 0 = not yet. Watchdog thread only.</summary>
public long SoftReportedAt { get; set; }
}
/// <summary>One loaded script: its compiled evaluator and its input slot order.</summary>
private sealed class LoadedScript : IDisposable