fix(galaxy): confirm unconfirmed native alarm acks against Galaxy's acknowledged transition (#869)

mxaccessgw answers AcknowledgeAlarm OK both when the AckMsg write was applied and when it was only sent (no OnWriteComplete; the reply then carries a diagnostic). EvaluateReply treated both as success, so the H6 live gate saw a condition go Acknowledged while Galaxy never applied it. - EvaluateReply returns Applied (no diagnostic) or Unconfirmed (diagnostic) and logs the gateway diagnostic on every branch that carries one. - GalaxyAckConfirmationTracker watches the alarm feed from before the RPC and, for an unconfirmed reply, waits up to AlarmAck.ConfirmTimeoutMs (default 10 s, 0 = old behaviour) for an Acknowledge/ActiveAcknowledged transition; on timeout it refuses (BadTimeout + diagnostic). - New optional IAlarmAcknowledgeConfirmation: DriverInstanceActor awaits it after AcknowledgeAsync, off its mailbox and outside the resilience pipeline; a failure is the existing #760 AlarmAckFailed revert. - AdminUI Galaxy form carries the new knob; docs updated. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 13:03:04 -05:00
parent e857135e8b
commit 4e492e6550
19 changed files with 1046 additions and 34 deletions
@@ -109,6 +109,11 @@
                           placeholder="secret:otopcua/galaxy/ack-fallback-password" />
                <div class="form-text">Forms: <code>secret:galaxy/NAME</code> or <code>secret:otopcua/galaxy/NAME</code> (production), <code>env:OTOPCUA_GALAXY_NAME</code>, <code>dev:VALUE</code>. A bare literal, <code>file:</code>, or a reference outside the Galaxy namespace is refused (#815).</div>
            </div>
+            <div class="col-md-3">
+                <label class="form-label">Ack confirm timeout (ms)</label>
+                <InputNumber @bind-Value="_form.Galaxy.AckConfirmTimeoutMs" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
+                <div class="form-text">When the gateway sends an ack but cannot confirm it, wait this long for Galaxy's acknowledged transition before reverting the condition to unacknowledged (#869). Default 10000; 0 = do not wait.</div>
+            </div>
        </div>
    </div>
 </section>
@@ -270,6 +275,7 @@
        public string? AckFallbackUser { get; set; }
        public string? AckFallbackDomain { get; set; }
        public string? AckFallbackPasswordSecretRef { get; set; }
+        public int AckConfirmTimeoutMs { get; set; } = GalaxyAlarmAckOptions.DefaultConfirmTimeoutMs;

        public static GalaxyFormModel FromRecord(GalaxyDriverOptions r)
        {
@@ -300,6 +306,7 @@
                AckFallbackUser = ack.FallbackUser,
                AckFallbackDomain = ack.FallbackDomain,
                AckFallbackPasswordSecretRef = ack.FallbackPasswordSecretRef,
+                AckConfirmTimeoutMs = ack.ConfirmTimeoutMs,
            };
        }

@@ -329,7 +336,10 @@
            AlarmAck = new GalaxyAlarmAckOptions(
                FallbackUser: string.IsNullOrWhiteSpace(AckFallbackUser) ? null : AckFallbackUser.Trim(),
                FallbackDomain: string.IsNullOrWhiteSpace(AckFallbackDomain) ? null : AckFallbackDomain.Trim(),
-                FallbackPasswordSecretRef: string.IsNullOrWhiteSpace(AckFallbackPasswordSecretRef) ? null : AckFallbackPasswordSecretRef.Trim()),
+                FallbackPasswordSecretRef: string.IsNullOrWhiteSpace(AckFallbackPasswordSecretRef) ? null : AckFallbackPasswordSecretRef.Trim())
+            {
+                ConfirmTimeoutMs = AckConfirmTimeoutMs,
+            },
        };
    }
 }
@@ -365,6 +365,10 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
    /// shutdown rather than let it race the teardown on a restart.</summary>
    private CancellationTokenSource? _initCts;

+    /// <summary>Cancelled by <see cref="PostStop"/> to abandon in-flight native-ack confirmations (lmxopcua#869);
+    /// never disposed — detached confirmation tasks may still hold its token.</summary>
+    private readonly CancellationTokenSource _ackConfirmCts = new();
+
    /// <summary>
    ///     Timestamps of recent Faulted-state transitions; used to compute the 5-minute error count.
    ///     No lock needed — every read/write site runs inside an Akka message handler, which is
@@ -1367,6 +1371,13 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
                cts.Token);
            _log.Info("DriverInstance {Id}: acknowledged native condition {Cond} by {User}",
                _driverInstanceId, msg.ConditionId, msg.OperatorUser);
+
+            // lmxopcua#869: an upstream may accept an ack without confirming it was applied (Galaxy: the gateway's
+            // "sent but unconfirmed" OK). The confirmation wait is seconds long, so it runs OFF this mailbox and
+            // OUTSIDE the resilience pipeline (a wait on an upstream event — it must neither be cut by the
+            // AlarmAcknowledge capability timeout nor count toward the breaker). Failure reverts like any other.
+            if (src is IAlarmAcknowledgeConfirmation confirmer)
+                _ = RunAckConfirmationAsync(confirmer, request, msg, replyTo, _ackConfirmCts.Token, _driverInstanceId, _log);
        }
        catch (DriverOperationRefusedException ex)
        {
@@ -1385,6 +1396,48 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
        }
    }

+    /// <summary>
+    ///     Backstop over a driver's own confirmation window (lmxopcua#869): the driver bounds the wait itself; this
+    ///     only guarantees a driver that ignores its window cannot hold a confirmation forever.
+    /// </summary>
+    internal static readonly TimeSpan AckConfirmationCeiling = TimeSpan.FromMinutes(3);
+
+    /// <summary>
+    ///     Awaits <see cref="IAlarmAcknowledgeConfirmation.ConfirmAcknowledgedAsync"/> off the actor context and, when
+    ///     the upstream does not confirm, reports <see cref="AlarmAckFailed"/> so the host reverts the condition to
+    ///     unacknowledged — the same report an <see cref="IAlarmSource.AcknowledgeAsync"/> failure makes. Reads no actor
+    ///     state: everything it needs is captured by the caller on the actor thread.
+    /// </summary>
+    private static async Task RunAckConfirmationAsync(
+        IAlarmAcknowledgeConfirmation confirmer, IReadOnlyList<AlarmAcknowledgeRequest> request, RouteAlarmAck msg,
+        IActorRef replyTo, CancellationToken stopToken, string driverInstanceId, ILoggingAdapter log)
+    {
+        await Task.Yield();
+        try
+        {
+            await confirmer.ConfirmAcknowledgedAsync(request, stopToken).WaitAsync(AckConfirmationCeiling, stopToken);
+        }
+        catch (DriverOperationRefusedException ex)
+        {
+            // The driver's verdict (e.g. Galaxy: no acknowledged transition inside its window) — it names why.
+            log.Warning("DriverInstance {Id}: native-alarm acknowledge of {Cond} by {User} was not confirmed upstream — {Reason}",
+                driverInstanceId, msg.ConditionId, msg.OperatorUser, ex.Message);
+            ReportAckFailed(replyTo, msg, ex.Message);
+        }
+        catch (OperationCanceledException) when (stopToken.IsCancellationRequested)
+        {
+            log.Info("DriverInstance {Id}: confirmation of the native-alarm acknowledge of {Cond} abandoned — the driver stopped",
+                driverInstanceId, msg.ConditionId);
+            ReportAckFailed(replyTo, msg, "the driver stopped before the upstream confirmed the acknowledge");
+        }
+        catch (Exception ex)
+        {
+            log.Warning(ex, "DriverInstance {Id}: confirmation of the native-alarm acknowledge of {Cond} failed",
+                driverInstanceId, msg.ConditionId);
+            ReportAckFailed(replyTo, msg, $"the upstream acknowledge was not confirmed ({ex.GetType().Name})");
+        }
+    }
+
    /// <summary>Tells <paramref name="replyTo"/> an <see cref="AlarmAckFailed"/> when the ack names a condition
    /// NodeId (lmxopcua#760); a no-op for an ack routed without one.</summary>
    private static void ReportAckFailed(IActorRef replyTo, RouteAlarmAck msg, string reason)
@@ -2004,6 +2057,10 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
        try { _initCts?.Cancel(); }
        catch (ObjectDisposedException) { /* the attempt already finished */ }

+        // In-flight native-ack confirmations (lmxopcua#869) are abandoned, and each reports its ack unconfirmed so
+        // the host reverts it — this incarnation can no longer observe the upstream's confirmation.
+        _ackConfirmCts.Cancel();
+
        // BOUNDED by ONE shared budget across shutdown + dispose. PostStop runs on a dispatcher thread and this
        // blocks it, so an unbounded wait on a driver whose shutdown hangs against a dead socket / COM proxy pinned
        // that thread for the whole wait, and a few such stops starved the dispatcher. A shutdown that overruns is