Merge #869 Galaxy native alarm ack: unconfirmed gateway OK waits for Galaxy's acknowledged transition, reverts on timeout
Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -42,10 +42,15 @@ yet re-listed. Only an identity written straight into the store by the `secret`
|
||||
refused until the next refresh, which the refusal itself requests. (2) anything else (a keyed session the cache cannot vouch for, MXAccess
|
||||
non-zero `hresult`, driver still connecting/reconnecting) comes back as `DriverInstanceActor.AlarmAckFailed` — or, for a
|
||||
disabled/unspawned/stubbed driver, is decided in the host itself (#765) — and `DriverHostActor` re-projects the condition unacknowledged — only while the ack is still pending in
|
||||
`NativeAlarmProjector` (any driver transition since then wins). A host that came up **Stale** cannot route any
|
||||
`NativeAlarmProjector` (a driver transition since then that STATES the ack wins; a Clear / same-message Raise only carries it through and leaves it revertible, #869; the revert is keyed to that ack's `MarkAcknowledged` sequence, echoed on `RouteAlarmAck`/`AlarmAckFailed`, so another ack's failure never undoes it). A host that came up **Stale** cannot route any
|
||||
native ack, so it marks the whole `NativeAlarmAckPreflightView` unavailable and every native ack is refused
|
||||
synchronously until `BecomeSteady` (#765). A test that routes an ack and expects it to STAND must start the owning
|
||||
driver (`AckingAlarmDriver` in Runtime.Tests).
|
||||
**A returned `AcknowledgeAsync` is not proof either (#869):** a driver implementing `IAlarmAcknowledgeConfirmation`
|
||||
(Galaxy — the gateway's OK also covers "AckMsg write sent, unconfirmed") is asked to confirm AFTER it, off the
|
||||
`DriverInstanceActor` mailbox and outside the resilience pipeline; a confirmation failure is the same
|
||||
`AlarmAckFailed` revert. Never await that confirmation inside the ack handler (`ReceiveAsync` suspends the mailbox)
|
||||
or inside the invoker (Tier-A 5 s timeout + breaker).
|
||||
|
||||
Behaviours that were wrong or missing and are now fixed — keep them: the SDK's `OnTimedUnshelve` is a
|
||||
**no-op** (the engine's `AutoUnshelve` owns timed-shelve expiry, and `UnshelveTime` reports the real
|
||||
|
||||
+16
-4
@@ -474,7 +474,12 @@ from the scripted `AlarmCommandRouter`:
|
||||
condition event). An ack whose owning driver is not running (disabled,
|
||||
not yet spawned) or is a stub on this node is reverted the same way by
|
||||
the host itself (#765). Only an ack still pending in `NativeAlarmProjector` is
|
||||
reverted: a driver transition projected since then is authoritative. A
|
||||
reverted: a driver transition projected since then that states the ack
|
||||
(Acknowledge, ActiveAcknowledged, or an un-acking Raise/Retrigger/
|
||||
InactiveUnacknowledged) is authoritative; one that only carries the ack
|
||||
through (a Clear, a same-message Raise) leaves it revertible (#869) — and
|
||||
only by a failure of THAT acknowledge: the revert is keyed to the sequence
|
||||
`MarkAcknowledged` issued, echoed on `RouteAlarmAck` / `AlarmAckFailed`. A
|
||||
refusal the driver can decide without I/O (Galaxy, with no fallback ack
|
||||
account: an anonymous session, or a keyed session whose key the cached
|
||||
`GalaxyAckIdentityDirectory` knows has no ack identity — #765) is instead
|
||||
@@ -483,9 +488,16 @@ from the scripted `AlarmCommandRouter`:
|
||||
before anything is applied — as is every native ack while the driver host
|
||||
is Stale (configuration database unreachable at boot), when no ack can be
|
||||
routed or reverted (#765). Galaxy value writes have a separate correlated
|
||||
write-outcome path. An awaited
|
||||
ack RPC does not prove the ack reached AVEVA event history: the H6 live
|
||||
verification remains [operator-gated](plans/2026-06-15-stillpending-phase-3-opcua-standards-design.md).
|
||||
write-outcome path. **An OK ack reply is not proof (#869):** the gateway
|
||||
also answers OK when the `AckMsg` write was only *sent* (its diagnostic
|
||||
says so). The driver then waits — off the actor mailbox, outside the
|
||||
resilience pipeline, bounded by `AlarmAck.ConfirmTimeoutMs` (default 10 s)
|
||||
— for Galaxy's own acknowledged transition on the alarm feed
|
||||
(`IAlarmAcknowledgeConfirmation`); without one the ack is reverted exactly
|
||||
as above and the `BadTimeout` verdict plus the gateway diagnostic are
|
||||
logged. See [`drivers/Galaxy.md` §Ack confirmation](drivers/Galaxy.md#ack-confirmation-869).
|
||||
The H6 live verification of a confirmed ack reaching AVEVA event history
|
||||
remains [operator-gated](plans/2026-06-15-stillpending-phase-3-opcua-standards-design.md).
|
||||
|
||||
Only the **Acknowledge** is routed to the driver. **A native condition
|
||||
supports Acknowledge and AddComment only** (Gitea #587 review): `Confirm`,
|
||||
|
||||
+47
-3
@@ -85,7 +85,7 @@ Project root files:
|
||||
- **`MxAccess`** — `ClientName` (must be unique per OtOpcUa instance — redundancy pairs enforce uniqueness at install time), `PublishingIntervalMs` (forwarded as `buffered_update_interval_ms` on subscribe), `WriteUserId` for ArchestrA secured-write, `EventPumpChannelCapacity` (default 50_000 — one second of headroom at 50k tags / 1Hz; tune via the `galaxy.events.dropped` metric).
|
||||
- **`Repository`** — `DiscoverPageSize`, `WatchDeployEvents`.
|
||||
- **`Reconnect`** — `InitialBackoffMs`, `MaxBackoffMs`, `ReplayOnSessionLost` (default `true`: after a gateway session is recreated, re-issue `SubscribeBulk` for every tracked subscription; `false` leaves them stranded — Bad + Degraded, see [Reconnect + Replay](#reconnect--replay)).
|
||||
- **`AlarmAck`** (optional, lmxopcua#749) — `FallbackUser`, `FallbackDomain`, `FallbackPasswordSecretRef`. See [Alarm acknowledge credential](#alarm-acknowledge-credential).
|
||||
- **`AlarmAck`** (optional, lmxopcua#749) — `FallbackUser`, `FallbackDomain`, `FallbackPasswordSecretRef`. See [Alarm acknowledge credential](#alarm-acknowledge-credential). `ConfirmTimeoutMs` (default `10000`, max `120000`, `0` = do not wait, negative = default; #869) — see [Ack confirmation](#ack-confirmation-869).
|
||||
|
||||
### Alarm acknowledge credential
|
||||
|
||||
@@ -129,8 +129,52 @@ Galaxy password, so `GalaxyAckCredentialResolver` picks the account per acknowle
|
||||
identity is broken (a user with no password secret), or MXAccess rejecting the account (a non-zero
|
||||
native `hresult`, e.g. a wrong password, now thrown as a refusal): the call returned `Good`, then
|
||||
`DriverHostActor` re-projects the condition **unacknowledged** (a fresh condition event,
|
||||
`Retain=true`) and logs the reason. A driver transition projected after the ack wins over a late
|
||||
failure.
|
||||
`Retain=true`) and logs the reason. A driver transition projected after the ack that states the ack
|
||||
state wins over a late failure; a Clear only carries it through and leaves it revertible (#869).
|
||||
|
||||
### Ack confirmation (#869)
|
||||
|
||||
The gateway answers `AcknowledgeAlarm` **OK with `hresult` 0 in two cases**: the `AckMsg` write's
|
||||
`OnWriteComplete` reported success (*applied* — empty `diagnostic_message`), or the write was sent and no
|
||||
completion arrived inside the worker's window (*sent but unconfirmed* — `diagnostic_message` says so,
|
||||
e.g. `AckMsg write to '…' sent; OnWriteComplete not observed within 5000 ms.`). The H6 live gate
|
||||
(2026-11-11, VD03) caught the second case: OtOpcUa showed the condition Acknowledged while Galaxy's
|
||||
`.Acked` stayed False and no ACK row was ever historized.
|
||||
|
||||
- `GatewayGalaxyAlarmAcknowledger.EvaluateReply` returns **Applied** for an OK reply with no diagnostic and
|
||||
**Unconfirmed** for one with a diagnostic (any unexplained diagnostic counts — the safe direction), and
|
||||
logs the gateway diagnostic on every branch that carries one.
|
||||
- `GalaxyAckConfirmationTracker` registers a watch for the alarm **before** the RPC (the alarm feed can
|
||||
deliver the confirming transition ahead of the unary reply) and, for an unconfirmed reply, waits up to
|
||||
`AlarmAck.ConfirmTimeoutMs` for an `Acknowledge` / `ActiveAcknowledged` transition of that alarm on the
|
||||
alarm feed (live or snapshot re-statement; the reference match is case-insensitive). Whoever acked it,
|
||||
Galaxy now shows it acknowledged — that is the truth the condition must match.
|
||||
- The wait is the optional `IAlarmAcknowledgeConfirmation` capability. `DriverInstanceActor` calls it after
|
||||
`AcknowledgeAsync` returned, **off its mailbox** (a detached task) and **outside the resilience pipeline**
|
||||
(a wait on an upstream event: it must not be cut by the Tier-A 5 s `AlarmAcknowledge` timeout nor count
|
||||
toward the breaker). The OPC UA method thread and the node-manager `Lock` are never involved — the method
|
||||
already returned when the fire-and-forget route started.
|
||||
- **On timeout** the confirmation throws `DriverOperationRefusedException` (`… not confirm … (BadTimeout;
|
||||
gateway: <diagnostic>)`), the driver logs a Warning with the diagnostic, and the actor reports
|
||||
`AlarmAckFailed`: the host re-projects the condition **unacknowledged** — the same #760 revert path.
|
||||
**What the OPC UA client sees:** the `Acknowledge` call has already returned `Good` (it cannot be changed
|
||||
after the fact, exactly as for every other #760 revert); the failure reaches the client as a fresh
|
||||
condition event with `AckedState = false`, and the `BadTimeout` verdict is recorded in the driver and host
|
||||
Warning logs. A confirming transition that arrives after the revert simply re-acknowledges the condition.
|
||||
A confirmation in flight when the driver actor stops is abandoned and reverted the same way.
|
||||
- **Limits to know:**
|
||||
- A gateway **reconnect** detaches the feed; an ack in flight across it is confirmed only if the new feed's
|
||||
snapshot re-statement (`ActiveAcknowledged`) arrives inside the window, otherwise it reverts (fails safe).
|
||||
- Acking an alarm that already **returned to normal** (`UNACK_RTN`) is confirmed by the worker's
|
||||
`Acknowledge` (`ACK_RTN`) transition; if the gateway reconciles it as a `Clear` instead, nothing confirms
|
||||
and the correctly-applied ack reverts to unacked. Unverified on hardware — include it in the next live gate.
|
||||
- The RPC itself still runs under the actor's 5 s ack bound and the Tier-A `AlarmAcknowledge` timeout (5 s).
|
||||
The gateway worker's write-completion window (`DefaultWriteCompletionTimeout`, 1.5 s) plus `AuthenticateUser` must stay
|
||||
well under that, or the unconfirmed reply is never seen and the ack reverts as a timeout that also counts
|
||||
toward the breaker.
|
||||
- `ConfirmTimeoutMs = 0` restores the pre-#869 behaviour (an unconfirmed ack counts as acknowledged, with a
|
||||
Warning) — an escape hatch only. Why the gateway-side ack did not stick on VD03 is tracked upstream
|
||||
(mxaccessgw#226).
|
||||
|
||||
The comment written to `AckMsg` is suffixed with the OPC UA identity —
|
||||
`"<comment> [OPC UA: <principal>, key <keyId>]"` or `"[OPC UA: anonymous]"` — so the client is not
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Optional companion to <see cref="IAlarmSource"/> for a driver whose upstream can ACCEPT an acknowledge
|
||||
/// without CONFIRMING it was applied (lmxopcua#869). mxaccessgw answers its <c>AcknowledgeAlarm</c> RPC OK
|
||||
/// both when the <c>AckMsg</c> write was applied and when it was only sent with no completion observed; the
|
||||
/// truth arrives later, as the alarm's own acknowledged transition on the alarm stream.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The runtime calls <see cref="ConfirmAcknowledgedAsync"/> after <see cref="IAlarmSource.AcknowledgeAsync"/>
|
||||
/// returned normally, with the SAME request instances, OUTSIDE the resilience pipeline (it is a wait on an
|
||||
/// upstream event, not I/O, so it must neither be cut by the capability timeout nor count toward the
|
||||
/// breaker) and OFF the driver actor's mailbox (the wait is seconds long).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// It completes normally when every request is confirmed, or never needed confirming (the upstream
|
||||
/// reported it applied). It throws — normally a <see cref="DriverOperationRefusedException"/> naming why —
|
||||
/// when an acknowledge is not confirmed within the driver's own bounded window; the runtime then reverts
|
||||
/// the condition to unacknowledged exactly as it does for an <see cref="IAlarmSource.AcknowledgeAsync"/>
|
||||
/// failure (lmxopcua#760). The window is the driver's to own and bound.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IAlarmAcknowledgeConfirmation
|
||||
{
|
||||
/// <summary>Waits for the upstream to confirm the acknowledges <see cref="IAlarmSource.AcknowledgeAsync"/> sent.</summary>
|
||||
/// <param name="acknowledgements">The request instances previously passed to <see cref="IAlarmSource.AcknowledgeAsync"/>.</param>
|
||||
/// <param name="cancellationToken">Abandons the wait (the caller then treats the acknowledge as unconfirmed).</param>
|
||||
/// <returns>A task that completes when every acknowledge is confirmed or needed no confirmation.</returns>
|
||||
Task ConfirmAcknowledgedAsync(
|
||||
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -79,6 +79,28 @@ public sealed record GalaxyAlarmAckOptions(
|
||||
string? FallbackDomain = null,
|
||||
string? FallbackPasswordSecretRef = null)
|
||||
{
|
||||
/// <summary>Default <see cref="ConfirmTimeoutMs"/>.</summary>
|
||||
public const int DefaultConfirmTimeoutMs = 10_000;
|
||||
|
||||
/// <summary>Upper bound for <see cref="ConfirmTimeoutMs"/>.</summary>
|
||||
public const int MaxConfirmTimeoutMs = 120_000;
|
||||
|
||||
/// <summary>
|
||||
/// How long an acknowledge the gateway SENT but could not confirm may wait for Galaxy's own acknowledged
|
||||
/// transition on the alarm feed before it is treated as failed and the OPC UA condition is reverted to
|
||||
/// unacknowledged (lmxopcua#869). Default 10 000 ms; <c>0</c> disables confirmation (an unconfirmed ack
|
||||
/// counts as success, with a Warning — the pre-#869 behaviour); a negative value means the default, and
|
||||
/// anything above 120 000 is clamped to it.
|
||||
/// </summary>
|
||||
[Display(Name = "Ack confirm timeout (ms)", Description = "Wait for Galaxy to confirm an unconfirmed acknowledge. 0 = do not wait.", GroupName = "Alarms")]
|
||||
[Range(0, MaxConfirmTimeoutMs)]
|
||||
public int ConfirmTimeoutMs { get; init; } = DefaultConfirmTimeoutMs;
|
||||
|
||||
/// <summary>The confirmation window: negative ⇒ the default, else clamped to <see cref="MaxConfirmTimeoutMs"/>.</summary>
|
||||
[System.Text.Json.Serialization.JsonIgnore]
|
||||
public TimeSpan ConfirmTimeout => TimeSpan.FromMilliseconds(
|
||||
ConfirmTimeoutMs < 0 ? DefaultConfirmTimeoutMs : Math.Min(ConfirmTimeoutMs, MaxConfirmTimeoutMs));
|
||||
|
||||
/// <summary>True when a fallback account is configured.</summary>
|
||||
[System.Text.Json.Serialization.JsonIgnore]
|
||||
public bool HasFallback => !string.IsNullOrWhiteSpace(FallbackUser);
|
||||
|
||||
@@ -38,7 +38,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy;
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class GalaxyDriver
|
||||
: IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IRediscoverable, IHostConnectivityProbe, IAlarmSource, IAlarmAcknowledgePreflight, ISkippedTagReporter, IDriverDiagnostics, IDisposable, IAsyncDisposable
|
||||
: IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IRediscoverable, IHostConnectivityProbe, IAlarmSource, IAlarmAcknowledgePreflight, IAlarmAcknowledgeConfirmation, ISkippedTagReporter, IDriverDiagnostics, IDisposable, IAsyncDisposable
|
||||
{
|
||||
// ⚠️ lmxopcua#815 warn-only compatibility (deprecated): this RUNNING driver keeps resolving a pre-#815
|
||||
// out-of-namespace credential reference (API key on InitializeAsync AND every gateway ReopenAsync, plus the
|
||||
@@ -138,6 +138,15 @@ public sealed class GalaxyDriver
|
||||
// lmxopcua#765: the cached key → has-Galaxy-identity set the synchronous ack pre-check reads (no I/O).
|
||||
// Null = no directory wired, so a keyed session is never pre-refused (the pre-#765 behaviour).
|
||||
private readonly IGalaxyAckIdentityDirectory? _ackIdentityDirectory;
|
||||
// lmxopcua#869: confirms acks the gateway SENT but could not confirm, against Galaxy's own acknowledged
|
||||
// transition on the alarm feed (bounded by AlarmAck.ConfirmTimeoutMs). Built in the ctor.
|
||||
private readonly GalaxyAckConfirmationTracker _ackConfirmations;
|
||||
|
||||
/// <summary>The ack confirmation window in force (lmxopcua#869) — tests.</summary>
|
||||
internal TimeSpan AckConfirmationWindow => _ackConfirmations.Window;
|
||||
|
||||
/// <summary>Acks still awaiting (or exempt from) confirmation (lmxopcua#869) — tests.</summary>
|
||||
internal int PendingAckConfirmations => _ackConfirmations.PendingCount;
|
||||
private IGalaxyAlarmFeed? _alarmFeed;
|
||||
private readonly Lock _alarmHandlersLock = new();
|
||||
private readonly Lock _alarmFeedLock = new();
|
||||
@@ -377,6 +386,8 @@ public sealed class GalaxyDriver
|
||||
_alarmFeed = alarmFeed;
|
||||
_ackCredentialResolver = ackCredentialResolver
|
||||
?? new GalaxyAckCredentialResolver(_driverInstanceId, _options.AlarmAck, _secretResolver, _logger, legacy: _legacyCredentials);
|
||||
_ackConfirmations = new GalaxyAckConfirmationTracker(
|
||||
_options.AlarmAck?.ConfirmTimeout ?? GalaxyAckConfirmationTracker.DefaultWindow, _logger, _driverInstanceId);
|
||||
|
||||
// Build the v3 RawPath ⇄ attributeRef tables from the authored raw tags. Empty when no tags
|
||||
// are authored (live-browse path), leaving every translation an identity no-op.
|
||||
@@ -2623,11 +2634,40 @@ public sealed class GalaxyDriver
|
||||
_driverInstanceId, alarmFullReference, credential.DisplayName, credential.Source,
|
||||
PrincipalOf(ack), ack.ApiKeyId ?? "none");
|
||||
|
||||
await _alarmAcknowledger.AcknowledgeAsync(
|
||||
alarmFullReference, comment, credential, cancellationToken).ConfigureAwait(false);
|
||||
// lmxopcua#869: the gateway's OK can mean "sent but unconfirmed". Watch for Galaxy's own acknowledged
|
||||
// transition from BEFORE the RPC — the feed can deliver it ahead of the unary reply — and arm the
|
||||
// watch when the reply is unconfirmed; ConfirmAcknowledgedAsync then awaits it, bounded.
|
||||
var watch = _ackConfirmations.Begin(ack, alarmFullReference);
|
||||
var armed = false;
|
||||
try
|
||||
{
|
||||
var outcome = await _alarmAcknowledger.AcknowledgeAsync(
|
||||
alarmFullReference, comment, credential, cancellationToken).ConfigureAwait(false);
|
||||
if (!outcome.IsApplied)
|
||||
{
|
||||
_ackConfirmations.Arm(watch, outcome.Diagnostic);
|
||||
armed = true;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!armed) _ackConfirmations.Release(watch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// lmxopcua#869: completes at once for an ack the gateway reported applied; for one it only SENT, waits up to
|
||||
/// <c>AlarmAck.ConfirmTimeoutMs</c> (default 10 s) for Galaxy's acknowledged transition on the alarm feed and
|
||||
/// throws <see cref="DriverOperationRefusedException"/> when it does not come — the host then reverts the
|
||||
/// condition to unacknowledged. The feed must be running (it is whenever the host holds native conditions,
|
||||
/// the only way an ack reaches this driver); without it nothing can confirm and the ack reverts.
|
||||
/// </remarks>
|
||||
public Task ConfirmAcknowledgedAsync(
|
||||
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken) =>
|
||||
_ackConfirmations.ConfirmAsync(acknowledgements, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Only cases decidable without I/O, on a driver with no <c>AlarmAck.FallbackUser</c>: an anonymous session
|
||||
@@ -2687,6 +2727,10 @@ public sealed class GalaxyDriver
|
||||
/// </summary>
|
||||
private void OnAlarmFeedTransition(object? sender, GalaxyAlarmTransition transition)
|
||||
{
|
||||
// lmxopcua#869: an acknowledged transition confirms any ack the gateway sent unconfirmed — before the
|
||||
// subscription gate below, which only governs fan-out to OnAlarmEvent.
|
||||
_ackConfirmations.Observe(transition);
|
||||
|
||||
GalaxyAlarmSubscriptionHandle? handle;
|
||||
int subCount;
|
||||
lock (_alarmHandlersLock)
|
||||
|
||||
@@ -114,7 +114,10 @@ public static class GalaxyDriverFactoryExtensions
|
||||
AlarmAck = new GalaxyAlarmAckOptions(
|
||||
FallbackUser: NullIfBlank(dto.AlarmAck?.FallbackUser),
|
||||
FallbackDomain: NullIfBlank(dto.AlarmAck?.FallbackDomain),
|
||||
FallbackPasswordSecretRef: NullIfBlank(dto.AlarmAck?.FallbackPasswordSecretRef)),
|
||||
FallbackPasswordSecretRef: NullIfBlank(dto.AlarmAck?.FallbackPasswordSecretRef))
|
||||
{
|
||||
ConfirmTimeoutMs = dto.AlarmAck?.ConfirmTimeoutMs ?? GalaxyAlarmAckOptions.DefaultConfirmTimeoutMs,
|
||||
},
|
||||
};
|
||||
|
||||
return new GalaxyDriver(
|
||||
@@ -160,6 +163,8 @@ public static class GalaxyDriverFactoryExtensions
|
||||
public string? FallbackDomain { get; init; }
|
||||
/// <summary>Gets or sets the fallback password reference (secret:/env:/file:/dev:).</summary>
|
||||
public string? FallbackPasswordSecretRef { get; init; }
|
||||
/// <summary>Gets or sets how long an unconfirmed ack waits for Galaxy to confirm it (lmxopcua#869); null = default.</summary>
|
||||
public int? ConfirmTimeoutMs { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>Gateway configuration section.</summary>
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Confirms Galaxy acknowledges the gateway sent but could not confirm (lmxopcua#869) against Galaxy's own
|
||||
/// acknowledged transition on the session-less alarm feed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Lifecycle of one acknowledge: <see cref="Begin"/> registers a watch BEFORE the gateway RPC is sent, so a
|
||||
/// confirming transition that overtakes the RPC reply is not lost (the feed and the unary reply race). The
|
||||
/// RPC outcome then either releases the watch (<see cref="Release"/>: applied, or the RPC failed) or arms it
|
||||
/// (<see cref="Arm"/>: sent but unconfirmed — the confirmation window starts). <see cref="ConfirmAsync"/>
|
||||
/// awaits the armed watches of the same request instances, outside the resilience pipeline and off the
|
||||
/// driver actor's mailbox (see <see cref="IAlarmAcknowledgeConfirmation"/>), and throws a
|
||||
/// <see cref="DriverOperationRefusedException"/> when the window passes without confirmation — the host
|
||||
/// then reverts the condition to unacknowledged.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A confirmation is an <see cref="GalaxyAlarmTransitionKind.Acknowledge"/> or
|
||||
/// <see cref="GalaxyAlarmTransitionKind.ActiveAcknowledged"/> transition (live or snapshot re-statement) for
|
||||
/// the same alarm reference — whoever acknowledged it, Galaxy now shows it acknowledged, which is the
|
||||
/// truth the OPC UA condition must match. Galaxy references are case-insensitive, so the match is too.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A watch whose <see cref="ConfirmAsync"/> never comes (an <see cref="IAlarmSource.AcknowledgeAsync"/>
|
||||
/// caller that does not confirm) is pruned once it is well past its window, so the set stays bounded.
|
||||
/// A zero window disables confirmation: an unconfirmed ack is counted as success with a Warning — the
|
||||
/// pre-#869 behaviour, kept only as an operator escape hatch.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class GalaxyAckConfirmationTracker
|
||||
{
|
||||
/// <summary>Default confirmation window — long enough for a gateway alarm-feed round trip, short enough
|
||||
/// that an operator sees the revert while still looking at the alarm.</summary>
|
||||
internal static readonly TimeSpan DefaultWindow = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>How long past its window an armed watch nobody confirms is kept before it is pruned.</summary>
|
||||
private static readonly TimeSpan AbandonedGrace = TimeSpan.FromMinutes(1);
|
||||
|
||||
private readonly TimeSpan _window;
|
||||
private readonly ILogger _logger;
|
||||
private readonly string _driverInstanceId;
|
||||
private readonly Lock _lock = new();
|
||||
private readonly List<Watch> _watches = [];
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="GalaxyAckConfirmationTracker"/> class.</summary>
|
||||
/// <param name="window">How long an unconfirmed ack may wait for Galaxy's transition; zero disables confirmation.</param>
|
||||
/// <param name="logger">Logger for confirmation outcomes.</param>
|
||||
/// <param name="driverInstanceId">The owning driver, for log context.</param>
|
||||
public GalaxyAckConfirmationTracker(TimeSpan window, ILogger logger, string driverInstanceId)
|
||||
{
|
||||
if (window < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(window), window, "must not be negative");
|
||||
_window = window;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_driverInstanceId = driverInstanceId ?? string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>The confirmation window (zero = disabled).</summary>
|
||||
public TimeSpan Window => _window;
|
||||
|
||||
/// <summary>Number of live watches (tests / diagnostics).</summary>
|
||||
internal int PendingCount
|
||||
{
|
||||
get { lock (_lock) return _watches.Count; }
|
||||
}
|
||||
|
||||
/// <summary>Registers a watch for <paramref name="alarmFullReference"/> before its acknowledge is sent.</summary>
|
||||
/// <param name="request">The request instance the caller will later pass to <see cref="ConfirmAsync"/>.</param>
|
||||
/// <param name="alarmFullReference">The Galaxy alarm reference the ack is sent for.</param>
|
||||
/// <returns>The watch, to be <see cref="Release"/>d or <see cref="Arm"/>ed once the RPC answers.</returns>
|
||||
public Watch Begin(AlarmAcknowledgeRequest request, string alarmFullReference)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentException.ThrowIfNullOrEmpty(alarmFullReference);
|
||||
var watch = new Watch(request, alarmFullReference);
|
||||
lock (_lock)
|
||||
{
|
||||
PruneAbandoned();
|
||||
_watches.Add(watch);
|
||||
}
|
||||
return watch;
|
||||
}
|
||||
|
||||
/// <summary>Drops a watch whose acknowledge needs no confirmation (applied) or was never delivered (failed).</summary>
|
||||
/// <param name="watch">The watch.</param>
|
||||
public void Release(Watch watch)
|
||||
{
|
||||
lock (_lock) _watches.Remove(watch);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts the confirmation window for an acknowledge the gateway sent but could not confirm. A watch the feed
|
||||
/// already confirmed while the RPC was in flight is released at once; with confirmation disabled the ack is
|
||||
/// counted as success (logged).
|
||||
/// </summary>
|
||||
/// <param name="watch">The watch.</param>
|
||||
/// <param name="diagnostic">The gateway's diagnostic, carried into the timeout report.</param>
|
||||
public void Arm(Watch watch, string diagnostic)
|
||||
{
|
||||
watch.Diagnostic = diagnostic ?? string.Empty;
|
||||
if (_window == TimeSpan.Zero)
|
||||
{
|
||||
Release(watch);
|
||||
_logger.LogWarning(
|
||||
"Galaxy driver {DriverInstanceId}: acknowledge of {AlarmRef} was not confirmed by the gateway ({Diagnostic}) "
|
||||
+ "and ack confirmation is disabled (AlarmAck.ConfirmTimeoutMs = 0); counting it as acknowledged.",
|
||||
_driverInstanceId, watch.AlarmFullReference, watch.Diagnostic);
|
||||
return;
|
||||
}
|
||||
|
||||
if (watch.IsConfirmed)
|
||||
{
|
||||
Release(watch);
|
||||
LogConfirmed(watch);
|
||||
return;
|
||||
}
|
||||
|
||||
lock (_lock) watch.ArmedTimestamp = Stopwatch.GetTimestamp();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Feed hook: confirms every watch for the transition's alarm when the transition says Galaxy shows it
|
||||
/// acknowledged. Runs on the feed's dispatch thread — never blocks.
|
||||
/// </summary>
|
||||
/// <param name="transition">The alarm transition.</param>
|
||||
public void Observe(GalaxyAlarmTransition transition)
|
||||
{
|
||||
if (transition.TransitionKind is not (GalaxyAlarmTransitionKind.Acknowledge or GalaxyAlarmTransitionKind.ActiveAcknowledged))
|
||||
return;
|
||||
if (string.IsNullOrEmpty(transition.AlarmFullReference)) return;
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
foreach (var watch in _watches)
|
||||
{
|
||||
if (string.Equals(watch.AlarmFullReference, transition.AlarmFullReference, StringComparison.OrdinalIgnoreCase))
|
||||
watch.Confirm();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Awaits confirmation of every armed watch belonging to <paramref name="requests"/> (matched by instance).
|
||||
/// Requests with no armed watch (applied, or never sent) need nothing. Throws
|
||||
/// <see cref="DriverOperationRefusedException"/> when any is not confirmed inside the window.
|
||||
/// </summary>
|
||||
/// <param name="requests">The request instances the acknowledge was sent with.</param>
|
||||
/// <param name="cancellationToken">Abandons the wait.</param>
|
||||
/// <returns>A task that completes when every armed watch is confirmed.</returns>
|
||||
public async Task ConfirmAsync(IReadOnlyList<AlarmAcknowledgeRequest> requests, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(requests);
|
||||
List<Watch> mine;
|
||||
lock (_lock)
|
||||
{
|
||||
mine = _watches.Where(w => w.ArmedTimestamp != 0 && requests.Any(r => ReferenceEquals(r, w.Request))).ToList();
|
||||
}
|
||||
if (mine.Count == 0) return;
|
||||
|
||||
var failures = new List<string>();
|
||||
try
|
||||
{
|
||||
foreach (var watch in mine)
|
||||
{
|
||||
var remaining = _window - Stopwatch.GetElapsedTime(watch.ArmedTimestamp);
|
||||
try
|
||||
{
|
||||
if (!watch.IsConfirmed)
|
||||
{
|
||||
if (remaining <= TimeSpan.Zero) throw new TimeoutException();
|
||||
await watch.Confirmed.WaitAsync(remaining, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
LogConfirmed(watch);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Galaxy driver {DriverInstanceId}: acknowledge of {AlarmRef} was NOT confirmed — Galaxy reported no "
|
||||
+ "acknowledged transition within {Window} after the gateway sent it unconfirmed (gateway diagnostic: "
|
||||
+ "{Diagnostic}). Treating it as failed (BadTimeout); the OPC UA condition is reverted to unacknowledged.",
|
||||
_driverInstanceId, watch.AlarmFullReference, _window, watch.Diagnostic);
|
||||
failures.Add(
|
||||
$"Galaxy did not confirm the acknowledge of '{watch.AlarmFullReference}' within "
|
||||
+ $"{_window.TotalSeconds:0.#} s (BadTimeout; gateway: {watch.Diagnostic})");
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
foreach (var watch in mine) _watches.Remove(watch);
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.Count > 0)
|
||||
throw new DriverOperationRefusedException(string.Join("; ", failures));
|
||||
}
|
||||
|
||||
private void LogConfirmed(Watch watch) =>
|
||||
_logger.LogInformation(
|
||||
"Galaxy driver {DriverInstanceId}: acknowledge of {AlarmRef} confirmed by Galaxy's acknowledged transition "
|
||||
+ "(the gateway had reported it sent but unconfirmed: {Diagnostic}).",
|
||||
_driverInstanceId, watch.AlarmFullReference, watch.Diagnostic);
|
||||
|
||||
// Caller holds _lock.
|
||||
private void PruneAbandoned()
|
||||
{
|
||||
var limit = _window + AbandonedGrace;
|
||||
_watches.RemoveAll(w => w.ArmedTimestamp != 0 && Stopwatch.GetElapsedTime(w.ArmedTimestamp) > limit);
|
||||
}
|
||||
|
||||
/// <summary>One acknowledge awaiting (or exempt from) confirmation.</summary>
|
||||
internal sealed class Watch
|
||||
{
|
||||
private readonly TaskCompletionSource _confirmed = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
internal Watch(AlarmAcknowledgeRequest request, string alarmFullReference)
|
||||
{
|
||||
Request = request;
|
||||
AlarmFullReference = alarmFullReference;
|
||||
}
|
||||
|
||||
/// <summary>The request instance this watch belongs to.</summary>
|
||||
public AlarmAcknowledgeRequest Request { get; }
|
||||
|
||||
/// <summary>The Galaxy alarm reference.</summary>
|
||||
public string AlarmFullReference { get; }
|
||||
|
||||
/// <summary>The gateway's diagnostic for the unconfirmed send.</summary>
|
||||
public string Diagnostic { get; set; } = string.Empty;
|
||||
|
||||
/// <summary><see cref="Stopwatch"/> timestamp the window started at; 0 while not armed.</summary>
|
||||
public long ArmedTimestamp { get; set; }
|
||||
|
||||
/// <summary>Completes when Galaxy reports the alarm acknowledged.</summary>
|
||||
public Task Confirmed => _confirmed.Task;
|
||||
|
||||
/// <summary>True once Galaxy reported the alarm acknowledged.</summary>
|
||||
public bool IsConfirmed => _confirmed.Task.IsCompleted;
|
||||
|
||||
/// <summary>Marks the watch confirmed (idempotent).</summary>
|
||||
public void Confirm() => _confirmed.TrySetResult();
|
||||
}
|
||||
}
|
||||
+44
-8
@@ -34,6 +34,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
/// gateway answered — so it neither retries nor counts toward the resilience breaker (a mis-keyed
|
||||
/// account must not open the breaker for every correctly-configured caller).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>An OK reply is not proof the ack was applied (lmxopcua#869).</b> The gateway answers OK with
|
||||
/// <c>hresult</c> 0 both when the <c>AckMsg</c> write completed and when it was only SENT — its worker saw no
|
||||
/// <c>OnWriteComplete</c> inside its window — and in the second case it sets <c>diagnostic_message</c>
|
||||
/// ("AckMsg write to '…' sent; OnWriteComplete not observed within N ms."). An applied ack carries an empty
|
||||
/// diagnostic. So an OK reply with a non-empty diagnostic is returned as
|
||||
/// <see cref="GalaxyAckOutcome.Unconfirmed"/> and the driver waits for Galaxy's own acknowledged transition
|
||||
/// (<see cref="GalaxyAckConfirmationTracker"/>). Any unexplained diagnostic on an OK reply is treated the
|
||||
/// same way — the safe direction: at worst an applied ack waits for its (arriving) transition. The H6 live
|
||||
/// gate (2026-11-11) caught an unconfirmed ack being reported Acknowledged while Galaxy never applied it.
|
||||
/// The diagnostic is logged on every branch that carries one.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class GatewayGalaxyAlarmAcknowledger : IGalaxyAlarmAcknowledger
|
||||
{
|
||||
@@ -53,7 +65,7 @@ internal sealed class GatewayGalaxyAlarmAcknowledger : IGalaxyAlarmAcknowledger
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task AcknowledgeAsync(
|
||||
public async Task<GalaxyAckOutcome> AcknowledgeAsync(
|
||||
string alarmFullReference,
|
||||
string comment,
|
||||
GalaxyAckCredential credential,
|
||||
@@ -77,7 +89,7 @@ internal sealed class GatewayGalaxyAlarmAcknowledger : IGalaxyAlarmAcknowledger
|
||||
+ "details withheld because they echoed the operator credential.");
|
||||
}
|
||||
|
||||
EvaluateReply(reply, alarmFullReference, credential, _logger);
|
||||
return EvaluateReply(reply, alarmFullReference, credential, _logger);
|
||||
}
|
||||
|
||||
/// <summary>Maps an acknowledge onto the gateway request. Pure — pinned by unit tests.</summary>
|
||||
@@ -104,23 +116,32 @@ internal sealed class GatewayGalaxyAlarmAcknowledger : IGalaxyAlarmAcknowledger
|
||||
|
||||
/// <summary>
|
||||
/// Applies the reply semantics: non-OK protocol status throws; a non-zero native code warns and throws a
|
||||
/// <see cref="DriverOperationRefusedException"/> (lmxopcua#760).
|
||||
/// Every message is scrubbed of the password.
|
||||
/// <see cref="DriverOperationRefusedException"/> (lmxopcua#760); an OK reply is
|
||||
/// <see cref="GalaxyAckOutcome.Applied"/> when it carries no diagnostic and
|
||||
/// <see cref="GalaxyAckOutcome.Unconfirmed"/> when it does (lmxopcua#869). The gateway's diagnostic is
|
||||
/// logged on every branch that carries one. Every message is scrubbed of the password.
|
||||
/// </summary>
|
||||
/// <param name="reply">The gateway reply.</param>
|
||||
/// <param name="alarmFullReference">The alarm reference.</param>
|
||||
/// <param name="credential">The credential the request carried.</param>
|
||||
/// <param name="logger">The logger.</param>
|
||||
internal static void EvaluateReply(
|
||||
/// <returns>Whether the gateway confirmed the ack was applied.</returns>
|
||||
internal static GalaxyAckOutcome EvaluateReply(
|
||||
AcknowledgeAlarmReply reply, string alarmFullReference, GalaxyAckCredential credential, ILogger logger)
|
||||
{
|
||||
var diagnostic = Scrub(reply.DiagnosticMessage, credential.Password).Trim();
|
||||
|
||||
// Protocol status — the gateway failed before MXAccess saw the ack. This is a
|
||||
// hard failure: the operator's request was not delivered at all.
|
||||
if (reply.ProtocolStatus is { } proto && proto.Code != ProtocolStatusCode.Ok)
|
||||
{
|
||||
var protoMessage = Scrub(proto.Message, credential.Password);
|
||||
logger.LogWarning(
|
||||
"Galaxy AcknowledgeAlarm for {AlarmRef} as {GalaxyUser} failed at the gateway: {ProtocolCode} {ProtocolMessage} (gateway diagnostic: {Diagnostic})",
|
||||
alarmFullReference, credential.DisplayName, proto.Code, protoMessage, DiagnosticText(diagnostic));
|
||||
throw new InvalidOperationException(
|
||||
$"Galaxy AcknowledgeAlarm for '{alarmFullReference}' as '{credential.DisplayName}' failed at the gateway: "
|
||||
+ $"{proto.Code} {Scrub(proto.Message, credential.Password)}");
|
||||
+ $"{proto.Code} {protoMessage}");
|
||||
}
|
||||
|
||||
// hresult is the authoritative native ack return code (0 = success) — the AuthenticateUser HRESULT
|
||||
@@ -129,14 +150,29 @@ internal sealed class GatewayGalaxyAlarmAcknowledger : IGalaxyAlarmAcknowledger
|
||||
if (reply.HasHresult && reply.Hresult != 0)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Galaxy AcknowledgeAlarm for {AlarmRef} as {GalaxyUser} returned native ack failure code {Hresult}.",
|
||||
alarmFullReference, credential.DisplayName, reply.Hresult);
|
||||
"Galaxy AcknowledgeAlarm for {AlarmRef} as {GalaxyUser} returned native ack failure code {Hresult} (gateway diagnostic: {Diagnostic}).",
|
||||
alarmFullReference, credential.DisplayName, reply.Hresult, DiagnosticText(diagnostic));
|
||||
throw new DriverOperationRefusedException(
|
||||
$"MXAccess rejected the Galaxy acknowledge of '{alarmFullReference}' as '{credential.DisplayName}' "
|
||||
+ $"(native code 0x{reply.Hresult:X8}); check the account's password and Galaxy acknowledge permission.");
|
||||
}
|
||||
|
||||
// lmxopcua#869: OK + hresult 0 means "applied OR sent-but-unconfirmed"; only the unconfirmed case carries a
|
||||
// diagnostic. Not a failure yet — the driver waits for Galaxy's own acknowledged transition.
|
||||
if (diagnostic.Length > 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Galaxy AcknowledgeAlarm for {AlarmRef} as {GalaxyUser} was sent but not confirmed by the gateway "
|
||||
+ "(gateway diagnostic: {Diagnostic}); awaiting Galaxy's acknowledged transition.",
|
||||
alarmFullReference, credential.DisplayName, diagnostic);
|
||||
return GalaxyAckOutcome.Unconfirmed(diagnostic);
|
||||
}
|
||||
|
||||
return GalaxyAckOutcome.Applied;
|
||||
}
|
||||
|
||||
private static string DiagnosticText(string diagnostic) => diagnostic.Length > 0 ? diagnostic : "none";
|
||||
|
||||
/// <summary>Replaces every occurrence of <paramref name="secret"/> in <paramref name="text"/>.</summary>
|
||||
/// <param name="text">Text that may echo the secret.</param>
|
||||
/// <param name="secret">The secret; empty means nothing to scrub.</param>
|
||||
|
||||
@@ -22,10 +22,32 @@ internal interface IGalaxyAlarmAcknowledger
|
||||
/// Carries a password — implementations must keep it out of logs and exception text.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">Cancels the gateway RPC.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task AcknowledgeAsync(
|
||||
/// <returns>
|
||||
/// Whether the gateway reported the ack APPLIED or only SENT without confirmation (lmxopcua#869). A refusal
|
||||
/// or failure throws instead.
|
||||
/// </returns>
|
||||
Task<GalaxyAckOutcome> AcknowledgeAsync(
|
||||
string alarmFullReference,
|
||||
string comment,
|
||||
GalaxyAckCredential credential,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What the gateway said about an acknowledge it accepted (lmxopcua#869). mxaccessgw answers OK both when the
|
||||
/// <c>AckMsg</c> write's <c>OnWriteComplete</c> reported success (<see cref="Applied"/>) and when the write was
|
||||
/// sent but no completion arrived inside the worker's window (<see cref="Unconfirmed"/>) — its contract tells
|
||||
/// clients to watch the alarm stream for the real acknowledged transition in that case.
|
||||
/// </summary>
|
||||
/// <param name="IsApplied">True when the gateway confirmed the ack write was applied.</param>
|
||||
/// <param name="Diagnostic">The gateway's diagnostic message (password-scrubbed); empty when it sent none.</param>
|
||||
internal sealed record GalaxyAckOutcome(bool IsApplied, string Diagnostic)
|
||||
{
|
||||
/// <summary>The gateway confirmed the ack write.</summary>
|
||||
public static GalaxyAckOutcome Applied { get; } = new(true, string.Empty);
|
||||
|
||||
/// <summary>The gateway sent the ack write but could not confirm it.</summary>
|
||||
/// <param name="diagnostic">The gateway's diagnostic message.</param>
|
||||
/// <returns>The outcome.</returns>
|
||||
public static GalaxyAckOutcome Unconfirmed(string diagnostic) => new(false, diagnostic ?? string.Empty);
|
||||
}
|
||||
|
||||
+11
-1
@@ -109,6 +109,11 @@
|
||||
placeholder="secret:otopcua/galaxy/ack-fallback-password" />
|
||||
<div class="form-text">Forms: <code>secret:galaxy/NAME</code> or <code>secret:otopcua/galaxy/NAME</code> (production), <code>env:OTOPCUA_GALAXY_NAME</code>, <code>dev:VALUE</code>. A bare literal, <code>file:</code>, or a reference outside the Galaxy namespace is refused (#815).</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<label class="form-label">Ack confirm timeout (ms)</label>
|
||||
<InputNumber @bind-Value="_form.Galaxy.AckConfirmTimeoutMs" @bind-Value:after="EmitAsync" min="0" max="120000" class="form-control form-control-sm" />
|
||||
<div class="form-text">When the gateway sends an ack but cannot confirm it, wait this long for Galaxy's acknowledged transition before reverting the condition to unacknowledged (#869). Default 10000; 0 = do not wait.</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
@@ -270,6 +275,7 @@
|
||||
public string? AckFallbackUser { get; set; }
|
||||
public string? AckFallbackDomain { get; set; }
|
||||
public string? AckFallbackPasswordSecretRef { get; set; }
|
||||
public int AckConfirmTimeoutMs { get; set; } = GalaxyAlarmAckOptions.DefaultConfirmTimeoutMs;
|
||||
|
||||
public static GalaxyFormModel FromRecord(GalaxyDriverOptions r)
|
||||
{
|
||||
@@ -300,6 +306,7 @@
|
||||
AckFallbackUser = ack.FallbackUser,
|
||||
AckFallbackDomain = ack.FallbackDomain,
|
||||
AckFallbackPasswordSecretRef = ack.FallbackPasswordSecretRef,
|
||||
AckConfirmTimeoutMs = ack.ConfirmTimeoutMs,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -329,7 +336,10 @@
|
||||
AlarmAck = new GalaxyAlarmAckOptions(
|
||||
FallbackUser: string.IsNullOrWhiteSpace(AckFallbackUser) ? null : AckFallbackUser.Trim(),
|
||||
FallbackDomain: string.IsNullOrWhiteSpace(AckFallbackDomain) ? null : AckFallbackDomain.Trim(),
|
||||
FallbackPasswordSecretRef: string.IsNullOrWhiteSpace(AckFallbackPasswordSecretRef) ? null : AckFallbackPasswordSecretRef.Trim()),
|
||||
FallbackPasswordSecretRef: string.IsNullOrWhiteSpace(AckFallbackPasswordSecretRef) ? null : AckFallbackPasswordSecretRef.Trim())
|
||||
{
|
||||
ConfirmTimeoutMs = AckConfirmTimeoutMs,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2413,7 +2413,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// projector too, or the next driver Clear projects (Active=false, Acked=prev=false) and un-acknowledges it.
|
||||
// Most sources (MTConnect, FOCAS) send no Acknowledge transition back, and the projector now survives a
|
||||
// deploy rather than resetting to its acked default.
|
||||
_nativeAlarmProjector.MarkAcknowledged(msg.ConditionNodeId);
|
||||
var ackSequence = _nativeAlarmProjector.MarkAcknowledged(msg.ConditionNodeId);
|
||||
|
||||
if (!_children.TryGetValue(target.DriverInstanceId, out var entry) || entry.Stubbed)
|
||||
{
|
||||
@@ -2424,7 +2424,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
var reason = entry is null ? "the owning driver is not running" : "the owning driver is stubbed on this node";
|
||||
_log.Debug("DriverHost {Node}: driver {Driver} cannot take the alarm ack of {Cond} ({Reason}) — dropped",
|
||||
_localNode, target.DriverInstanceId, msg.ConditionNodeId, reason);
|
||||
RevertNativeAlarmAck(msg.ConditionNodeId, reason);
|
||||
RevertNativeAlarmAck(msg.ConditionNodeId, reason, ackSequence);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2433,7 +2433,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// carried here) and reverts the node (HandleAlarmAckFailed). The driver correlates on ConditionId
|
||||
// (= the authored alarm FullName the inverse map keyed on).
|
||||
entry.Actor.Tell(new DriverInstanceActor.RouteAlarmAck(
|
||||
target.RawPath, msg.Comment, msg.OperatorUser, msg.ApiKeyId, ConditionNodeId: msg.ConditionNodeId));
|
||||
target.RawPath, msg.Comment, msg.OperatorUser, msg.ApiKeyId, ConditionNodeId: msg.ConditionNodeId,
|
||||
AckSequence: ackSequence));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -2444,7 +2445,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// </summary>
|
||||
/// <param name="msg">The failure report from the driver child.</param>
|
||||
private void HandleAlarmAckFailed(DriverInstanceActor.AlarmAckFailed msg) =>
|
||||
RevertNativeAlarmAck(msg.ConditionNodeId, msg.Reason);
|
||||
RevertNativeAlarmAck(msg.ConditionNodeId, msg.Reason, msg.AckSequence);
|
||||
|
||||
/// <summary>
|
||||
/// Re-projects <paramref name="conditionNodeId"/> as unacknowledged when its local acknowledge is still
|
||||
@@ -2453,12 +2454,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// </summary>
|
||||
/// <param name="conditionNodeId">The bare condition NodeId (Raw realm).</param>
|
||||
/// <param name="reason">Why the acknowledge did not reach the upstream alarm system.</param>
|
||||
private void RevertNativeAlarmAck(string conditionNodeId, string reason)
|
||||
/// <param name="ackSequence">The failed acknowledge's projector sequence (lmxopcua#869).</param>
|
||||
private void RevertNativeAlarmAck(string conditionNodeId, string reason, long ackSequence)
|
||||
{
|
||||
if (!_nativeAlarmProjector.TryRevertAcknowledge(conditionNodeId, out var snapshot) || snapshot is null)
|
||||
// Keyed to the failed ack's own sequence (lmxopcua#869): a pending ack from an earlier or later acknowledge of
|
||||
// the same condition is not this failure's to undo.
|
||||
if (!_nativeAlarmProjector.TryRevertAcknowledge(conditionNodeId, ackSequence, out var snapshot) || snapshot is null)
|
||||
{
|
||||
_log.Info("DriverHost {Node}: native-alarm acknowledge of {Cond} did not reach the alarm system ({Reason}); " +
|
||||
"the condition has since been re-stated by its driver, so it is left as projected",
|
||||
"no acknowledge of THIS request is pending (re-stated by its driver since, superseded by another ack, or it changed nothing), so it is left as projected",
|
||||
_localNode, conditionNodeId, reason);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -90,8 +90,11 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// acknowledge that does not reach the upstream alarm system (refused, failed, or dropped because the driver
|
||||
/// is not connected) is reported back to the sender as an <see cref="AlarmAckFailed"/> so the host can revert
|
||||
/// the locally-applied acknowledge (lmxopcua#760). Null ⇒ no report.</param>
|
||||
/// <param name="AckSequence">The host projector's sequence for this acknowledge (lmxopcua#869), echoed on
|
||||
/// <see cref="AlarmAckFailed"/> so only a failure of THIS ack reverts it; 0 ⇒ the ack changed nothing.</param>
|
||||
public sealed record RouteAlarmAck(
|
||||
string ConditionId, string? Comment, string OperatorUser, string? ApiKeyId = null, string? ConditionNodeId = null);
|
||||
string ConditionId, string? Comment, string OperatorUser, string? ApiKeyId = null, string? ConditionNodeId = null,
|
||||
long AckSequence = 0);
|
||||
|
||||
/// <summary>
|
||||
/// Reply to a <see cref="RouteAlarmAck"/> carrying a <see cref="RouteAlarmAck.ConditionNodeId"/> whose
|
||||
@@ -99,7 +102,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
/// <param name="ConditionNodeId">The OPC UA condition NodeId the client acknowledged.</param>
|
||||
/// <param name="Reason">Why the acknowledge did not go through; never contains a secret.</param>
|
||||
public sealed record AlarmAckFailed(string ConditionNodeId, string Reason);
|
||||
/// <param name="AckSequence">The failed acknowledge's <see cref="RouteAlarmAck.AckSequence"/> (lmxopcua#869).</param>
|
||||
public sealed record AlarmAckFailed(string ConditionNodeId, string Reason, long AckSequence = 0);
|
||||
public sealed record Subscribe(IReadOnlyList<string> FullReferences, TimeSpan PublishingInterval);
|
||||
/// <summary>
|
||||
/// Sets the set of references this driver should keep subscribed for the lifetime of the
|
||||
@@ -365,6 +369,10 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// shutdown rather than let it race the teardown on a restart.</summary>
|
||||
private CancellationTokenSource? _initCts;
|
||||
|
||||
/// <summary>Cancelled by <see cref="PostStop"/> to abandon in-flight native-ack confirmations (lmxopcua#869);
|
||||
/// never disposed — detached confirmation tasks may still hold its token.</summary>
|
||||
private readonly CancellationTokenSource _ackConfirmCts = new();
|
||||
|
||||
/// <summary>
|
||||
/// Timestamps of recent Faulted-state transitions; used to compute the 5-minute error count.
|
||||
/// No lock needed — every read/write site runs inside an Akka message handler, which is
|
||||
@@ -1367,6 +1375,21 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
cts.Token);
|
||||
_log.Info("DriverInstance {Id}: acknowledged native condition {Cond} by {User}",
|
||||
_driverInstanceId, msg.ConditionId, msg.OperatorUser);
|
||||
|
||||
// lmxopcua#869: an upstream may accept an ack without confirming it was applied (Galaxy: the gateway's
|
||||
// "sent but unconfirmed" OK). The confirmation wait is seconds long, so it runs OFF this mailbox and
|
||||
// OUTSIDE the resilience pipeline (a wait on an upstream event — it must neither be cut by the
|
||||
// AlarmAcknowledge capability timeout nor count toward the breaker). Failure reverts like any other.
|
||||
// Task.Run, not a bare call: this handler is a ReceiveAsync, so TaskScheduler.Current is the actor's
|
||||
// scheduler; the confirmation must not depend on this actor's context at all — in particular not after
|
||||
// PostStop, when it still has to report the abandoned ack (pinned by DriverInstanceActorAckConfirmationTests).
|
||||
if (src is IAlarmAcknowledgeConfirmation confirmer)
|
||||
{
|
||||
var stopToken = _ackConfirmCts.Token;
|
||||
var driverInstanceId = _driverInstanceId;
|
||||
var log = _log;
|
||||
_ = Task.Run(() => RunAckConfirmationAsync(confirmer, request, msg, replyTo, stopToken, driverInstanceId, log));
|
||||
}
|
||||
}
|
||||
catch (DriverOperationRefusedException ex)
|
||||
{
|
||||
@@ -1385,12 +1408,54 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Backstop over a driver's own confirmation window (lmxopcua#869): the driver bounds the wait itself; this
|
||||
/// only guarantees a driver that ignores its window cannot hold a confirmation forever.
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan AckConfirmationCeiling = TimeSpan.FromMinutes(3);
|
||||
|
||||
/// <summary>
|
||||
/// Awaits <see cref="IAlarmAcknowledgeConfirmation.ConfirmAcknowledgedAsync"/> off the actor context and, when
|
||||
/// the upstream does not confirm, reports <see cref="AlarmAckFailed"/> so the host reverts the condition to
|
||||
/// unacknowledged — the same report an <see cref="IAlarmSource.AcknowledgeAsync"/> failure makes. Reads no actor
|
||||
/// state: everything it needs is captured by the caller on the actor thread.
|
||||
/// </summary>
|
||||
private static async Task RunAckConfirmationAsync(
|
||||
IAlarmAcknowledgeConfirmation confirmer, IReadOnlyList<AlarmAcknowledgeRequest> request, RouteAlarmAck msg,
|
||||
IActorRef replyTo, CancellationToken stopToken, string driverInstanceId, ILoggingAdapter log)
|
||||
{
|
||||
try
|
||||
{
|
||||
await confirmer.ConfirmAcknowledgedAsync(request, stopToken)
|
||||
.WaitAsync(AckConfirmationCeiling, stopToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (DriverOperationRefusedException ex)
|
||||
{
|
||||
// The driver's verdict (e.g. Galaxy: no acknowledged transition inside its window) — it names why.
|
||||
log.Warning("DriverInstance {Id}: native-alarm acknowledge of {Cond} by {User} was not confirmed upstream — {Reason}",
|
||||
driverInstanceId, msg.ConditionId, msg.OperatorUser, ex.Message);
|
||||
ReportAckFailed(replyTo, msg, ex.Message);
|
||||
}
|
||||
catch (OperationCanceledException) when (stopToken.IsCancellationRequested)
|
||||
{
|
||||
log.Info("DriverInstance {Id}: confirmation of the native-alarm acknowledge of {Cond} abandoned — the driver stopped",
|
||||
driverInstanceId, msg.ConditionId);
|
||||
ReportAckFailed(replyTo, msg, "the driver stopped before the upstream confirmed the acknowledge");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
log.Warning(ex, "DriverInstance {Id}: confirmation of the native-alarm acknowledge of {Cond} failed",
|
||||
driverInstanceId, msg.ConditionId);
|
||||
ReportAckFailed(replyTo, msg, $"the upstream acknowledge was not confirmed ({ex.GetType().Name})");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Tells <paramref name="replyTo"/> an <see cref="AlarmAckFailed"/> when the ack names a condition
|
||||
/// NodeId (lmxopcua#760); a no-op for an ack routed without one.</summary>
|
||||
private static void ReportAckFailed(IActorRef replyTo, RouteAlarmAck msg, string reason)
|
||||
{
|
||||
if (string.IsNullOrEmpty(msg.ConditionNodeId) || replyTo.IsNobody()) return;
|
||||
replyTo.Tell(new AlarmAckFailed(msg.ConditionNodeId, reason));
|
||||
replyTo.Tell(new AlarmAckFailed(msg.ConditionNodeId, reason, msg.AckSequence));
|
||||
}
|
||||
|
||||
private async Task HandleSubscribeAsync(Subscribe msg)
|
||||
@@ -2004,6 +2069,10 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
try { _initCts?.Cancel(); }
|
||||
catch (ObjectDisposedException) { /* the attempt already finished */ }
|
||||
|
||||
// In-flight native-ack confirmations (lmxopcua#869) are abandoned, and each reports its ack unconfirmed so
|
||||
// the host reverts it — this incarnation can no longer observe the upstream's confirmation.
|
||||
_ackConfirmCts.Cancel();
|
||||
|
||||
// BOUNDED by ONE shared budget across shutdown + dispose. PostStop runs on a dispatcher thread and this
|
||||
// blocks it, so an unbounded wait on a driver whose shutdown hangs against a dead socket / COM proxy pinned
|
||||
// that thread for the whole wait, and a few such stops starved the dispatcher. A shutdown that overruns is
|
||||
|
||||
@@ -18,9 +18,13 @@ public sealed class NativeAlarmProjector
|
||||
/// <summary>
|
||||
/// Conditions a client acknowledged locally (<see cref="MarkAcknowledged"/>) whose acknowledge has not yet
|
||||
/// been confirmed or superseded by a driver transition — the only ones <see cref="TryRevertAcknowledge"/>
|
||||
/// may un-acknowledge (lmxopcua#760).
|
||||
/// may un-acknowledge (lmxopcua#760) — keyed to the sequence <see cref="MarkAcknowledged"/> issued, so only a
|
||||
/// failure of THAT acknowledge reverts it (lmxopcua#869).
|
||||
/// </summary>
|
||||
private readonly HashSet<string> _pendingLocalAck = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, long> _pendingLocalAck = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Last sequence issued by <see cref="MarkAcknowledged"/>; 0 is never issued.</summary>
|
||||
private long _ackSequence;
|
||||
|
||||
/// <summary>Project an alarm transition onto the full condition snapshot for <paramref name="nodeId"/>.</summary>
|
||||
/// <param name="nodeId">The materialised condition node's id (the projection's state key).</param>
|
||||
@@ -78,9 +82,6 @@ public sealed class NativeAlarmProjector
|
||||
/// <returns>The full Part 9 condition snapshot to write to the node.</returns>
|
||||
public AlarmConditionSnapshot Project(string nodeId, AlarmEventArgs e, out IReadOnlyList<AlarmTransitionKind> historyKinds)
|
||||
{
|
||||
// Any driver transition supersedes a pending local acknowledge: from here the projected state is the
|
||||
// driver's word, so a late ack failure must not rewrite it.
|
||||
_pendingLocalAck.Remove(nodeId);
|
||||
var prev = _prior.TryGetValue(nodeId, out var p)
|
||||
? p : (Active: false, Acked: true, Message: (string?)null, Severity: (ushort)0, StatusCode: OpcUaStatus.Good, Class: (AlarmConditionClass?)null);
|
||||
// A source with a finer scale than the four buckets supplies the exact Part 9 severity.
|
||||
@@ -153,6 +154,15 @@ public sealed class NativeAlarmProjector
|
||||
break;
|
||||
}
|
||||
|
||||
// A driver transition that STATES the ack (Acknowledge / ActiveAcknowledged, or an un-acking Raise /
|
||||
// Retrigger / InactiveUnacknowledged) supersedes a pending local acknowledge: from here the ack state is the
|
||||
// driver's word, so a late ack failure must not rewrite it. One that merely CARRIES the prior ack through
|
||||
// (a Clear, a same-message Raise, a message/severity update) says nothing about it — the optimistic local
|
||||
// ack rode along, so it stays pending and a later failure still reverts it (lmxopcua#869: an operator acks,
|
||||
// the value returns to normal while the unconfirmed ack is still being confirmed, Galaxy shows UNACK_RTN).
|
||||
if (acked != prev.Acked || e.Kind is AlarmTransitionKind.Acknowledge or AlarmTransitionKind.ActiveAcknowledged)
|
||||
_pendingLocalAck.Remove(nodeId);
|
||||
|
||||
var statusCode = e.QualityStatusCode ?? OpcUaStatus.Good;
|
||||
// A source that classifies its conditions (#839) states the class on its transitions; one that does not (or a
|
||||
// quality-only annotation) keeps whatever the condition last carried.
|
||||
@@ -174,27 +184,60 @@ public sealed class NativeAlarmProjector
|
||||
/// stale unacknowledged prior. An untracked condition already defaults to acknowledged.
|
||||
/// </summary>
|
||||
/// <param name="nodeId">The acknowledged condition node id.</param>
|
||||
public void MarkAcknowledged(string nodeId)
|
||||
/// <returns>
|
||||
/// The sequence identifying this pending acknowledge — pass it back to
|
||||
/// <see cref="TryRevertAcknowledge(string, long, out AlarmConditionSnapshot?)"/> with its failure — or 0 when the
|
||||
/// ack changed nothing (untracked, or already acknowledged), so its failure has nothing to undo.
|
||||
/// </returns>
|
||||
public long MarkAcknowledged(string nodeId)
|
||||
{
|
||||
if (!_prior.TryGetValue(nodeId, out var p)) return;
|
||||
if (!_prior.TryGetValue(nodeId, out var p)) return 0;
|
||||
// Only an ack that CHANGED the projected state is pending; re-acking an acked condition has nothing to undo.
|
||||
if (!p.Acked) _pendingLocalAck.Add(nodeId);
|
||||
long sequence = 0;
|
||||
if (!p.Acked)
|
||||
{
|
||||
sequence = ++_ackSequence;
|
||||
_pendingLocalAck[nodeId] = sequence;
|
||||
}
|
||||
_prior[nodeId] = p with { Acked = true };
|
||||
return sequence;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Undo a client acknowledgement the driver refused or failed to deliver upstream (lmxopcua#760), returning
|
||||
/// the unacknowledged snapshot to re-project onto the node. Only an acknowledge still pending since
|
||||
/// <see cref="MarkAcknowledged"/> is reverted: once any driver transition for the condition has been
|
||||
/// projected, that transition is authoritative and a late failure changes nothing.
|
||||
/// <see cref="MarkAcknowledged"/> is reverted: once a driver transition that STATES the condition's ack state
|
||||
/// has been projected, that transition is authoritative and a late failure changes nothing. A transition that
|
||||
/// only carries the prior ack through (e.g. a Clear) leaves it pending (lmxopcua#869).
|
||||
/// </summary>
|
||||
/// <param name="nodeId">The condition node id whose acknowledge failed.</param>
|
||||
/// <param name="snapshot">The unacknowledged snapshot to project; null when nothing was reverted.</param>
|
||||
/// <returns>True when the condition was reverted to unacknowledged.</returns>
|
||||
public bool TryRevertAcknowledge(string nodeId, out AlarmConditionSnapshot? snapshot)
|
||||
public bool TryRevertAcknowledge(string nodeId, out AlarmConditionSnapshot? snapshot) =>
|
||||
TryRevert(nodeId, ackSequence: null, out snapshot);
|
||||
|
||||
/// <summary>
|
||||
/// Like <see cref="TryRevertAcknowledge(string, out AlarmConditionSnapshot?)"/>, but only when the pending
|
||||
/// acknowledge is the one <see cref="MarkAcknowledged"/> issued <paramref name="ackSequence"/> for
|
||||
/// (lmxopcua#869). A pending ack can outlive its own delivery — a source that never sends an Acknowledge
|
||||
/// transition back (FOCAS, MTConnect), across a Clear that only carried it through — so a failure of a LATER,
|
||||
/// unrelated acknowledge (a re-ack while the driver is down) or of an EARLIER one (a Galaxy confirmation timing
|
||||
/// out after the alarm re-occurred and was acked again) must not revert it.
|
||||
/// </summary>
|
||||
/// <param name="nodeId">The condition node id whose acknowledge failed.</param>
|
||||
/// <param name="ackSequence">The sequence of the failed acknowledge; 0 never matches.</param>
|
||||
/// <param name="snapshot">The unacknowledged snapshot to project; null when nothing was reverted.</param>
|
||||
/// <returns>True when the condition was reverted to unacknowledged.</returns>
|
||||
public bool TryRevertAcknowledge(string nodeId, long ackSequence, out AlarmConditionSnapshot? snapshot) =>
|
||||
TryRevert(nodeId, ackSequence, out snapshot);
|
||||
|
||||
private bool TryRevert(string nodeId, long? ackSequence, out AlarmConditionSnapshot? snapshot)
|
||||
{
|
||||
snapshot = null;
|
||||
if (!_pendingLocalAck.Remove(nodeId) || !_prior.TryGetValue(nodeId, out var p)) return false;
|
||||
if (!_pendingLocalAck.TryGetValue(nodeId, out var pending)
|
||||
|| (ackSequence is { } wanted && wanted != pending)
|
||||
|| !_prior.TryGetValue(nodeId, out var p)) return false;
|
||||
_pendingLocalAck.Remove(nodeId);
|
||||
_prior[nodeId] = p with { Acked = false };
|
||||
snapshot = new AlarmConditionSnapshot(
|
||||
Active: p.Active, Acknowledged: false, Confirmed: true, Enabled: true,
|
||||
|
||||
@@ -317,10 +317,10 @@ public sealed class GalaxyDriverAlarmSourceTests
|
||||
/// <param name="credential">The resolved Galaxy credential.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A completed task.</returns>
|
||||
public Task AcknowledgeAsync(string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
public Task<GalaxyAckOutcome> AcknowledgeAsync(string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
{
|
||||
Calls.Add((alarmFullReference, comment, credential.User));
|
||||
return Task.CompletedTask;
|
||||
return Task.FromResult(GalaxyAckOutcome.Applied);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+408
@@ -0,0 +1,408 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 — mxaccessgw answers <c>AcknowledgeAlarm</c> OK both when the <c>AckMsg</c> write was APPLIED and
|
||||
/// when it was only SENT (no <c>OnWriteComplete</c> inside the worker's window; the reply then carries a
|
||||
/// diagnostic). The H6 live gate saw an unconfirmed ack reported Acknowledged while Galaxy never applied it.
|
||||
/// The driver now treats an unconfirmed OK as pending and confirms it against Galaxy's own acknowledged
|
||||
/// transition on the alarm feed, bounded by <c>AlarmAck.ConfirmTimeoutMs</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The "fake gateway" is a scripted <see cref="AcknowledgeAlarmReply"/> run through the production
|
||||
/// <see cref="GatewayGalaxyAlarmAcknowledger.EvaluateReply"/> — the gRPC transport seam is internal to the
|
||||
/// client package, so the reply mapping is the part under test.
|
||||
/// </remarks>
|
||||
public sealed class GalaxyAckConfirmationTests
|
||||
{
|
||||
private const string AlarmRef = "Tank01.Level.HiHi";
|
||||
|
||||
private const string UnconfirmedDiagnostic =
|
||||
"AckMsg write to 'Tank01.Level.HiHi.AckMsg' sent; OnWriteComplete not observed within 5000 ms.";
|
||||
|
||||
private static readonly GalaxyAckCredential Alice =
|
||||
new("alice", "CORP", "alice-pw", GalaxyAckCredentialSource.ApiKey);
|
||||
|
||||
// ------------------------------------------------------------------ reply mapping
|
||||
|
||||
[Fact]
|
||||
public void EvaluateReply_ok_without_diagnostic_is_applied()
|
||||
{
|
||||
var logger = new CapturingLogger();
|
||||
|
||||
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(AppliedReply(), AlarmRef, Alice, logger);
|
||||
|
||||
outcome.IsApplied.ShouldBeTrue();
|
||||
logger.Entries.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
/// <summary>The H6 reply: OK + hresult 0 + a "sent; not observed" diagnostic is UNCONFIRMED, and the
|
||||
/// diagnostic is logged. Before #869 the same reply returned silently as success.</summary>
|
||||
[Fact]
|
||||
public void EvaluateReply_ok_with_diagnostic_is_unconfirmed_and_logs_the_diagnostic()
|
||||
{
|
||||
var logger = new CapturingLogger();
|
||||
|
||||
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(UnconfirmedReply(), AlarmRef, Alice, logger);
|
||||
|
||||
outcome.IsApplied.ShouldBeFalse();
|
||||
outcome.Diagnostic.ShouldBe(UnconfirmedDiagnostic);
|
||||
logger.AllText.ShouldContain(UnconfirmedDiagnostic);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateReply_native_refusal_logs_the_diagnostic_and_refuses()
|
||||
{
|
||||
var reply = new AcknowledgeAlarmReply
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
Hresult = 1008,
|
||||
DiagnosticMessage = "AckMsg write to 'Tank01.Level.HiHi.AckMsg' rejected: category SecurityError, detail 1008.",
|
||||
};
|
||||
var logger = new CapturingLogger();
|
||||
|
||||
Should.Throw<DriverOperationRefusedException>(
|
||||
() => GatewayGalaxyAlarmAcknowledger.EvaluateReply(reply, AlarmRef, Alice, logger));
|
||||
|
||||
logger.AllText.ShouldContain("rejected: category SecurityError");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateReply_scrubs_the_password_from_the_diagnostic()
|
||||
{
|
||||
var reply = new AcknowledgeAlarmReply
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
Hresult = 0,
|
||||
DiagnosticMessage = "sent as alice/alice-pw; not observed",
|
||||
};
|
||||
var logger = new CapturingLogger();
|
||||
|
||||
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(reply, AlarmRef, Alice, logger);
|
||||
|
||||
outcome.Diagnostic.ShouldNotContain("alice-pw");
|
||||
logger.AllText.ShouldNotContain("alice-pw");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ driver: confirmation
|
||||
|
||||
/// <summary>Applied OK → acknowledged: nothing pending, the confirmation completes at once.</summary>
|
||||
[Fact]
|
||||
public async Task Applied_ack_is_confirmed_at_once()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(AppliedReply());
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 30_000);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
||||
|
||||
confirm.IsCompletedSuccessfully.ShouldBeTrue("an applied ack needs no confirmation");
|
||||
gateway.Calls.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unconfirmed OK + Galaxy's acknowledged transition on the feed → acknowledged, end to end through the
|
||||
/// driver's feed hook (non-matching transitions are emitted first; that they do NOT confirm is pinned
|
||||
/// synchronously by <see cref="Tracker_confirms_only_on_this_alarms_acknowledged_transition"/>).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Unconfirmed_ack_is_confirmed_by_the_acknowledged_transition()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(UnconfirmedReply());
|
||||
var logger = new CapturingLogger();
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 120_000, logger);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
||||
|
||||
feed.Emit(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
|
||||
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Raise));
|
||||
feed.Emit(Transition(AlarmRef.ToUpperInvariant(), GalaxyAlarmTransitionKind.Acknowledge));
|
||||
|
||||
await TestWait.ForAsync(confirm, "the acknowledged transition must confirm the pending ack");
|
||||
logger.AllText.ShouldContain(UnconfirmedDiagnostic);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The feed and the unary reply race: Galaxy's acknowledged transition can arrive BEFORE the RPC answers
|
||||
/// "unconfirmed". The watch is registered before the RPC, so that early transition still confirms — with a
|
||||
/// 1 ms window, a lost transition would time out instead.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Acknowledged_transition_that_overtakes_the_reply_still_confirms()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(UnconfirmedReply())
|
||||
{
|
||||
DuringCall = () => feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Acknowledge)),
|
||||
};
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 1);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
await driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unconfirmed OK + no acknowledged transition → the confirmation fails with a refusal naming the timeout and
|
||||
/// the gateway diagnostic (the runtime reverts the condition on it), and a Warning carrying the diagnostic is
|
||||
/// logged. Transitions that must not confirm (another alarm's ack, this alarm's retrigger) are emitted during
|
||||
/// the wait. Before #869 this ack counted as acknowledged.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Unconfirmed_ack_without_confirmation_fails_with_the_diagnostic()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(UnconfirmedReply());
|
||||
var logger = new CapturingLogger();
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 300, logger);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
||||
feed.Emit(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
|
||||
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Retrigger));
|
||||
|
||||
var ex = await Should.ThrowAsync<DriverOperationRefusedException>(() => confirm);
|
||||
ex.Message.ShouldContain(AlarmRef);
|
||||
ex.Message.ShouldContain("BadTimeout");
|
||||
ex.Message.ShouldContain(UnconfirmedDiagnostic);
|
||||
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Text.Contains(UnconfirmedDiagnostic));
|
||||
}
|
||||
|
||||
/// <summary>A refused ack (non-zero native code) throws from AcknowledgeAsync — the existing revert path —
|
||||
/// and leaves nothing pending to confirm.</summary>
|
||||
[Fact]
|
||||
public async Task Refused_ack_throws_and_leaves_nothing_pending()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(new AcknowledgeAlarmReply
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
Hresult = unchecked((int)0x80070005),
|
||||
});
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 30_000);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await Should.ThrowAsync<DriverOperationRefusedException>(
|
||||
() => driver.AcknowledgeAsync(request, CancellationToken.None));
|
||||
|
||||
driver.ConfirmAcknowledgedAsync(request, CancellationToken.None).IsCompletedSuccessfully.ShouldBeTrue();
|
||||
driver.PendingAckConfirmations.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary><c>ConfirmTimeoutMs = 0</c> is the opt-out: an unconfirmed ack counts as acknowledged, with a Warning.</summary>
|
||||
[Fact]
|
||||
public async Task Zero_timeout_counts_an_unconfirmed_ack_as_acknowledged_with_a_warning()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(UnconfirmedReply());
|
||||
var logger = new CapturingLogger();
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 0, logger);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
|
||||
driver.ConfirmAcknowledgedAsync(request, CancellationToken.None).IsCompletedSuccessfully.ShouldBeTrue();
|
||||
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Text.Contains("confirmation is disabled"));
|
||||
driver.PendingAckConfirmations.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>Each confirmation is for ITS request instance: confirming one leaves no watch behind, and a later
|
||||
/// ack of the same alarm is confirmed independently.</summary>
|
||||
[Fact]
|
||||
public async Task Confirmations_release_their_watches()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(UnconfirmedReply());
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 120_000);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
|
||||
for (var i = 0; i < 2; i++)
|
||||
{
|
||||
var request = Ack(AlarmRef);
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
driver.PendingAckConfirmations.ShouldBe(1);
|
||||
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
||||
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.ActiveAcknowledged));
|
||||
await TestWait.ForAsync(confirm, $"ack #{i} must be confirmed");
|
||||
driver.PendingAckConfirmations.ShouldBe(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tracker matching rules, checked synchronously: only an Acknowledge / ActiveAcknowledged transition of the
|
||||
/// SAME alarm (case-insensitive) confirms. Positive control: the matching transition does flip it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Tracker_confirms_only_on_this_alarms_acknowledged_transition()
|
||||
{
|
||||
var tracker = new GalaxyAckConfirmationTracker(TimeSpan.FromSeconds(30), new CapturingLogger(), "drv");
|
||||
var watch = tracker.Begin(Ack(AlarmRef)[0], AlarmRef);
|
||||
tracker.Arm(watch, UnconfirmedDiagnostic);
|
||||
|
||||
foreach (var kind in new[]
|
||||
{
|
||||
GalaxyAlarmTransitionKind.Raise, GalaxyAlarmTransitionKind.Clear,
|
||||
GalaxyAlarmTransitionKind.Retrigger, GalaxyAlarmTransitionKind.Unspecified,
|
||||
})
|
||||
{
|
||||
tracker.Observe(Transition(AlarmRef, kind));
|
||||
}
|
||||
tracker.Observe(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
|
||||
tracker.Observe(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.ActiveAcknowledged));
|
||||
watch.IsConfirmed.ShouldBeFalse("no transition so far says THIS alarm is acknowledged");
|
||||
|
||||
tracker.Observe(Transition(AlarmRef.ToLowerInvariant(), GalaxyAlarmTransitionKind.Acknowledge));
|
||||
watch.IsConfirmed.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Options_default_to_ten_seconds_and_map_negative_to_the_default()
|
||||
{
|
||||
new GalaxyAlarmAckOptions().ConfirmTimeout.ShouldBe(TimeSpan.FromSeconds(10));
|
||||
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = -5 }.ConfirmTimeout.ShouldBe(TimeSpan.FromSeconds(10));
|
||||
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 0 }.ConfirmTimeout.ShouldBe(TimeSpan.Zero);
|
||||
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 999_999 }.ConfirmTimeout.ShouldBe(TimeSpan.FromMinutes(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Factory_binds_ConfirmTimeoutMs_and_defaults_when_absent()
|
||||
{
|
||||
const string Json = """
|
||||
{ "gateway": { "endpoint": "https://gw:5001", "apiKeySecretRef": "dev:k" },
|
||||
"mxAccess": { "clientName": "C" }, "alarmAck": { "confirmTimeoutMs": 2500 } }
|
||||
""";
|
||||
using var withKnob = GalaxyDriverFactoryExtensions.CreateInstance("g", Json);
|
||||
withKnob.AckConfirmationWindow.ShouldBe(TimeSpan.FromMilliseconds(2500));
|
||||
|
||||
using var without = GalaxyDriverFactoryExtensions.CreateInstance(
|
||||
"g", """{ "gateway": { "endpoint": "https://gw:5001", "apiKeySecretRef": "dev:k" }, "mxAccess": { "clientName": "C" } }""");
|
||||
without.AckConfirmationWindow.ShouldBe(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ helpers
|
||||
|
||||
private static AcknowledgeAlarmReply AppliedReply() => new()
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
Hresult = 0,
|
||||
};
|
||||
|
||||
private static AcknowledgeAlarmReply UnconfirmedReply() => new()
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
Hresult = 0,
|
||||
DiagnosticMessage = UnconfirmedDiagnostic,
|
||||
};
|
||||
|
||||
private static AlarmAcknowledgeRequest[] Ack(string conditionId) =>
|
||||
[new AlarmAcknowledgeRequest(SourceNodeId: "Tank01", ConditionId: conditionId, Comment: "c")];
|
||||
|
||||
private static GalaxyDriver NewDriver(
|
||||
IGalaxyAlarmFeed feed, IGalaxyAlarmAcknowledger acknowledger, int confirmTimeoutMs, CapturingLogger? logger = null)
|
||||
{
|
||||
var options = new GalaxyDriverOptions(
|
||||
new GalaxyGatewayOptions("http://localhost:5000", "literal-api-key"),
|
||||
new GalaxyMxAccessOptions("AckConfirmTest"),
|
||||
new GalaxyRepositoryOptions(),
|
||||
new GalaxyReconnectOptions())
|
||||
{
|
||||
AlarmAck = new GalaxyAlarmAckOptions { ConfirmTimeoutMs = confirmTimeoutMs },
|
||||
};
|
||||
return new GalaxyDriver(
|
||||
driverInstanceId: "drv-ack",
|
||||
options: options,
|
||||
hierarchySource: null,
|
||||
alarmAcknowledger: acknowledger,
|
||||
alarmFeed: feed,
|
||||
logger: logger is null ? null : new TypedLogger<GalaxyDriver>(logger),
|
||||
ackCredentialResolver: FixedAckCredentialResolver.Fallback());
|
||||
}
|
||||
|
||||
private static GalaxyAlarmTransition Transition(string alarmRef, GalaxyAlarmTransitionKind kind) => new(
|
||||
AlarmFullReference: alarmRef,
|
||||
SourceObjectReference: "Tank01",
|
||||
AlarmTypeName: "AnalogLimitAlarm.HiHi",
|
||||
TransitionKind: kind,
|
||||
SeverityBucket: AlarmSeverity.High,
|
||||
OpcUaSeverity: 700,
|
||||
RawMxAccessSeverity: 500,
|
||||
OriginalRaiseTimestampUtc: null,
|
||||
TransitionTimestampUtc: DateTime.UtcNow,
|
||||
OperatorUser: string.Empty,
|
||||
OperatorComment: string.Empty,
|
||||
Category: "Process",
|
||||
Description: "Tank 01 high-high level");
|
||||
|
||||
/// <summary>
|
||||
/// A fake gateway: every call answers with <paramref name="reply"/>, mapped by the production
|
||||
/// <see cref="GatewayGalaxyAlarmAcknowledger.EvaluateReply"/>. <see cref="DuringCall"/> runs while the RPC is
|
||||
/// "in flight" (before the reply is evaluated).
|
||||
/// </summary>
|
||||
private sealed class ScriptedGateway(AcknowledgeAlarmReply reply) : IGalaxyAlarmAcknowledger
|
||||
{
|
||||
private int _calls;
|
||||
|
||||
public Action? DuringCall { get; init; }
|
||||
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
public Task<GalaxyAckOutcome> AcknowledgeAsync(
|
||||
string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _calls);
|
||||
DuringCall?.Invoke();
|
||||
try
|
||||
{
|
||||
return Task.FromResult(GatewayGalaxyAlarmAcknowledger.EvaluateReply(
|
||||
reply, alarmFullReference, credential, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return Task.FromException<GalaxyAckOutcome>(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeAlarmFeed : IGalaxyAlarmFeed
|
||||
{
|
||||
public event EventHandler<GalaxyAlarmTransition>? OnAlarmTransition;
|
||||
|
||||
public void Start() { }
|
||||
|
||||
public void Emit(GalaxyAlarmTransition transition) => OnAlarmTransition?.Invoke(this, transition);
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class TypedLogger<T>(CapturingLogger inner) : ILogger<T>
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(LogLevel l, EventId id, TState s, Exception? e, Func<TState, Exception?, string> f)
|
||||
{
|
||||
lock (_gate) inner.Log(l, id, s, e, f);
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -485,11 +485,11 @@ public sealed class GalaxyAckCredentialTests
|
||||
{
|
||||
public List<(string AlarmRef, string Comment, GalaxyAckCredential Credential)> Calls { get; } = [];
|
||||
|
||||
public Task AcknowledgeAsync(
|
||||
public Task<GalaxyAckOutcome> AcknowledgeAsync(
|
||||
string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
{
|
||||
Calls.Add((alarmFullReference, comment, credential));
|
||||
return Task.CompletedTask;
|
||||
return Task.FromResult(GalaxyAckOutcome.Applied);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -242,10 +242,10 @@ public sealed class GalaxyDriverRawPathKeyingTests
|
||||
{
|
||||
public List<(string AlarmRef, string Comment, string Operator)> Calls { get; } = [];
|
||||
|
||||
public Task AcknowledgeAsync(string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
public Task<GalaxyAckOutcome> AcknowledgeAsync(string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
{
|
||||
Calls.Add((alarmFullReference, comment, credential.User));
|
||||
return Task.CompletedTask;
|
||||
return Task.FromResult(GalaxyAckOutcome.Applied);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+8
-8
@@ -34,22 +34,22 @@ public sealed class GatewayGalaxyAlarmAcknowledgerTests
|
||||
/// </summary>
|
||||
private sealed class FakeAcknowledger : IGalaxyAlarmAcknowledger
|
||||
{
|
||||
private readonly Queue<Func<Task>> _outcomes = new();
|
||||
private readonly Queue<Func<Task<GalaxyAckOutcome>>> _outcomes = new();
|
||||
|
||||
/// <summary>Calls recorded in the order received.</summary>
|
||||
public List<(string AlarmRef, string Comment, string Operator)> Calls { get; } = [];
|
||||
|
||||
/// <summary>Enqueues an outcome for the next call (completes normally or throws).</summary>
|
||||
public void Enqueue(Func<Task> outcome) => _outcomes.Enqueue(outcome);
|
||||
public void Enqueue(Func<Task<GalaxyAckOutcome>> outcome) => _outcomes.Enqueue(outcome);
|
||||
|
||||
public Task AcknowledgeAsync(
|
||||
public Task<GalaxyAckOutcome> AcknowledgeAsync(
|
||||
string alarmFullReference,
|
||||
string comment,
|
||||
GalaxyAckCredential credential,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Calls.Add((alarmFullReference, comment, credential.User));
|
||||
return _outcomes.Count > 0 ? _outcomes.Dequeue()() : Task.CompletedTask;
|
||||
return _outcomes.Count > 0 ? _outcomes.Dequeue()() : Task.FromResult(GalaxyAckOutcome.Applied);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,9 +77,9 @@ public sealed class GatewayGalaxyAlarmAcknowledgerTests
|
||||
{
|
||||
var fake = new FakeAcknowledger();
|
||||
// First call: simulates a transient transport blip (e.g. RpcException from gRPC).
|
||||
fake.Enqueue(() => Task.FromException(new IOException("synthetic transport fault")));
|
||||
fake.Enqueue(() => Task.FromException<GalaxyAckOutcome>(new IOException("synthetic transport fault")));
|
||||
// Second call: succeeds — proves no dead-state latch.
|
||||
fake.Enqueue(() => Task.CompletedTask);
|
||||
fake.Enqueue(() => Task.FromResult(GalaxyAckOutcome.Applied));
|
||||
|
||||
var driver = new GalaxyDriver("g", Opts(),
|
||||
hierarchySource: null, alarmAcknowledger: fake,
|
||||
@@ -109,8 +109,8 @@ public sealed class GatewayGalaxyAlarmAcknowledgerTests
|
||||
public async Task Acknowledge_after_protocol_failure_still_issues_next_call()
|
||||
{
|
||||
var fake = new FakeAcknowledger();
|
||||
fake.Enqueue(() => Task.FromException(new InvalidOperationException("gateway overload")));
|
||||
fake.Enqueue(() => Task.CompletedTask);
|
||||
fake.Enqueue(() => Task.FromException<GalaxyAckOutcome>(new InvalidOperationException("gateway overload")));
|
||||
fake.Enqueue(() => Task.FromResult(GalaxyAckOutcome.Applied));
|
||||
|
||||
var driver = new GalaxyDriver("g", Opts(),
|
||||
hierarchySource: null, alarmAcknowledger: fake,
|
||||
|
||||
+26
@@ -259,4 +259,30 @@ public sealed class GalaxyDriverPageFormSerializationTests
|
||||
json.ShouldNotContain("hasFallback");
|
||||
JsonSerializer.Deserialize<GalaxyDriverOptions>(json, _pageOpts)!.AlarmAck.ShouldBe(original.AlarmAck);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869: the ack-confirmation window must survive FromRecord/ToRecord and JSON, or saving the page
|
||||
/// would silently reset an operator's setting (e.g. the 0 = do-not-wait opt-out) to the default.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void FormModel_RoundTrips_AlarmAckConfirmTimeout()
|
||||
{
|
||||
var original = new GalaxyDriverOptions(
|
||||
new GalaxyGatewayOptions("https://gw:5001", "env:K"),
|
||||
new GalaxyMxAccessOptions("OtOpcUa"),
|
||||
new GalaxyRepositoryOptions(),
|
||||
new GalaxyReconnectOptions())
|
||||
{
|
||||
AlarmAck = new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 0 },
|
||||
};
|
||||
|
||||
var back = GalaxyDriverForm.GalaxyFormModel.FromRecord(original).ToRecord();
|
||||
back.AlarmAck.ConfirmTimeoutMs.ShouldBe(0);
|
||||
|
||||
var json = JsonSerializer.Serialize(back, _pageOpts);
|
||||
json.ShouldContain("\"confirmTimeoutMs\":0");
|
||||
json.ShouldNotContain("\"confirmTimeout\":");
|
||||
JsonSerializer.Deserialize<GalaxyDriverOptions>(json, _pageOpts)!.AlarmAck.ConfirmTimeoutMs.ShouldBe(0);
|
||||
new GalaxyDriverForm.GalaxyFormModel().AckConfirmTimeoutMs.ShouldBe(GalaxyAlarmAckOptions.DefaultConfirmTimeoutMs);
|
||||
}
|
||||
}
|
||||
|
||||
+117
-1
@@ -217,11 +217,105 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
RaiseAndExpectUnacked(actor, publish);
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
|
||||
AwaitAssert(() => recorder.Acks.Count.ShouldBe(1), duration: Timeout, cancellationToken: TestContext.Current.CancellationToken);
|
||||
// lmxopcua#869: the driver is an IAlarmAcknowledgeConfirmation whose confirmation succeeds at once — it was
|
||||
// asked, and a confirmed ack must not revert.
|
||||
AwaitAssert(() => recorder.ConfirmCalls.ShouldBe(1), duration: Timeout, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
actor.Tell(new DriverInstanceActor.AttributeAlarmPublished("drv-1", Evt(AlarmTransitionKind.Clear)));
|
||||
var next = NextAlarmUpdate(publish);
|
||||
next.State.Active.ShouldBeFalse();
|
||||
next.State.Acknowledged.ShouldBeTrue("the successful ack must have survived — no revert");
|
||||
|
||||
// A Clear carries the ack through and so leaves a still-pending ack revertible (#869): a wrong revert landing
|
||||
// after it would be the NEXT update. A carry-through update must therefore still show it acknowledged.
|
||||
actor.Tell(new DriverInstanceActor.AttributeAlarmPublished("drv-1", Evt(AlarmTransitionKind.Unspecified)));
|
||||
NextAlarmUpdate(publish).State.Acknowledged.ShouldBeTrue("no late revert may follow a confirmed ack");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 — the common sequence: an operator acks, the alarm returns to normal (a Clear, carrying the
|
||||
/// optimistic ack) while the unconfirmed ack is still being confirmed, then the confirmation fails. The ack
|
||||
/// must still revert — to inactive + UNacknowledged, Galaxy's UNACK_RTN — not stay hidden behind the Clear.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void An_unconfirmed_ack_reverts_even_after_a_clear_inside_the_window()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var gate = new TaskCompletionSource();
|
||||
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway") { Confirmation = () => gate.Task };
|
||||
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
|
||||
var publish = this.CreateCleanOutcomePublishProbe();
|
||||
var actor = SpawnHostAndApply(db, deploymentId, recorder, publish.Ref);
|
||||
|
||||
RaiseAndExpectUnacked(actor, publish);
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
|
||||
AwaitAssert(() => recorder.ConfirmCalls.ShouldBe(1), duration: Timeout, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
actor.Tell(new DriverInstanceActor.AttributeAlarmPublished("drv-1", Evt(AlarmTransitionKind.Clear)));
|
||||
var cleared = NextAlarmUpdate(publish);
|
||||
cleared.State.Active.ShouldBeFalse();
|
||||
cleared.State.Acknowledged.ShouldBeTrue("the optimistic ack rides the Clear");
|
||||
|
||||
gate.SetException(new DriverOperationRefusedException("Galaxy did not confirm the acknowledge (BadTimeout)"));
|
||||
var reverted = NextAlarmUpdate(publish);
|
||||
reverted.State.Active.ShouldBeFalse();
|
||||
reverted.State.Acknowledged.ShouldBeFalse("the unconfirmed ack must not survive behind the Clear");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 — the driver ACCEPTED the ack (AcknowledgeAsync returned normally, e.g. the gateway's "sent but
|
||||
/// unconfirmed" OK) but its upstream never confirmed it: the confirmation refusal reverts the condition to
|
||||
/// unacknowledged exactly like an AcknowledgeAsync failure. Before #869 nothing awaited a confirmation, so the
|
||||
/// condition stayed Acknowledged although Galaxy never applied the ack.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void An_accepted_but_unconfirmed_ack_reverts_the_condition_to_unacknowledged()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway")
|
||||
{
|
||||
Confirmation = () => Task.FromException(new DriverOperationRefusedException(
|
||||
"Galaxy did not confirm the acknowledge of 'Tank.Hi' within 10 s (BadTimeout; gateway: AckMsg write sent; OnWriteComplete not observed)")),
|
||||
};
|
||||
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
|
||||
var publish = this.CreateCleanOutcomePublishProbe();
|
||||
var actor = SpawnHostAndApply(db, deploymentId, recorder, publish.Ref);
|
||||
|
||||
RaiseAndExpectUnacked(actor, publish);
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
|
||||
|
||||
var reverted = NextAlarmUpdate(publish);
|
||||
reverted.AlarmNodeId.ShouldBe(AlarmRawPath);
|
||||
reverted.State.Active.ShouldBeTrue();
|
||||
reverted.State.Acknowledged.ShouldBeFalse("an ack the upstream never confirmed must not stand");
|
||||
recorder.Acks.Count.ShouldBe(1);
|
||||
recorder.ConfirmCalls.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 — the confirmation wait runs OFF the driver actor's mailbox: while one ack's confirmation is
|
||||
/// still pending (it never completes here), a second ack is still routed to the driver. A confirmation awaited
|
||||
/// inside the ack handler would suspend the mailbox and the second ack would never arrive.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_pending_confirmation_does_not_block_the_driver_actor()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var never = new TaskCompletionSource();
|
||||
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway")
|
||||
{
|
||||
Confirmation = () => never.Task,
|
||||
};
|
||||
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
|
||||
var actor = SpawnHostAndApply(db, deploymentId, recorder);
|
||||
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "first", "alice"));
|
||||
AwaitAssert(() => recorder.ConfirmCalls.ShouldBe(1), duration: Timeout, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "second", "alice"));
|
||||
AwaitAssert(() => recorder.Acks.ShouldContain(a => a.Comment == "second"), duration: Timeout,
|
||||
cancellationToken: TestContext.Current.CancellationToken);
|
||||
never.TrySetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -452,6 +546,12 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
/// <summary>The driver's synchronous pre-check verdict (lmxopcua#760); null ⇒ always proceed.</summary>
|
||||
public Func<string?, string?>? PreflightRefusal { init => _driver.PreflightRefusal = value; }
|
||||
|
||||
/// <summary>lmxopcua#869: what <see cref="IAlarmAcknowledgeConfirmation.ConfirmAcknowledgedAsync"/> returns; null ⇒ confirmed at once.</summary>
|
||||
public Func<Task>? Confirmation { init => _driver.Confirmation = value; }
|
||||
|
||||
/// <summary>How many confirmations the spawned driver was asked for.</summary>
|
||||
public int ConfirmCalls => _driver.ConfirmCalls;
|
||||
|
||||
/// <inheritdoc />
|
||||
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
|
||||
{
|
||||
@@ -465,8 +565,24 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
}
|
||||
|
||||
/// <summary>An <see cref="IDriver"/> + <see cref="IAlarmSource"/> that records every acknowledge.</summary>
|
||||
private sealed class RecordingAlarmDriver : IDriver, IAlarmSource, IAlarmAcknowledgePreflight
|
||||
private sealed class RecordingAlarmDriver : IDriver, IAlarmSource, IAlarmAcknowledgePreflight, IAlarmAcknowledgeConfirmation
|
||||
{
|
||||
/// <summary>lmxopcua#869: the confirmation outcome; null ⇒ confirmed at once.</summary>
|
||||
public Func<Task>? Confirmation { get; set; }
|
||||
|
||||
private int _confirmCalls;
|
||||
|
||||
/// <summary>How many confirmations were requested.</summary>
|
||||
public int ConfirmCalls => Volatile.Read(ref _confirmCalls);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ConfirmAcknowledgedAsync(
|
||||
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _confirmCalls);
|
||||
return Confirmation?.Invoke() ?? Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>When set, <see cref="AcknowledgeAsync"/> records the request and then throws this.</summary>
|
||||
public Exception? AckFault { get; set; }
|
||||
|
||||
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 â <see cref="DriverInstanceActor"/> asks an <see cref="IAlarmAcknowledgeConfirmation"/> driver to
|
||||
/// confirm an ack AFTER <see cref="IAlarmSource.AcknowledgeAsync"/> returned, off its mailbox, and reports
|
||||
/// <see cref="DriverInstanceActor.AlarmAckFailed"/> to the router when the confirmation fails â including one that
|
||||
/// fails long after the ack handler returned, and one abandoned because the actor stopped.
|
||||
/// </summary>
|
||||
public sealed class DriverInstanceActorAckConfirmationTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly TimeSpan Timeout = PresenceBudget;
|
||||
|
||||
/// <summary>A confirmation that faults asynchronously, after the ack handler returned, is reported failed.</summary>
|
||||
[Fact]
|
||||
public void A_confirmation_that_fails_later_is_reported_as_a_failed_ack()
|
||||
{
|
||||
var driver = new ConfirmingDriver();
|
||||
var actor = SpawnConnected(driver);
|
||||
var gate = new TaskCompletionSource();
|
||||
driver.Confirmation = () => gate.Task;
|
||||
var router = CreateTestProbe();
|
||||
|
||||
actor.Tell(Ack("n1"), router.Ref);
|
||||
AwaitCondition(() => driver.ConfirmCalls == 1, Timeout, TestContext.Current.CancellationToken);
|
||||
gate.SetException(new DriverOperationRefusedException("not confirmed within 10 s (BadTimeout)"));
|
||||
|
||||
var failed = router.ExpectMsg<DriverInstanceActor.AlarmAckFailed>(Timeout, cancellationToken: TestContext.Current.CancellationToken);
|
||||
failed.ConditionNodeId.ShouldBe("n1");
|
||||
failed.Reason.ShouldContain("BadTimeout");
|
||||
driver.ForeignConfirmations.ShouldBe(0, "the confirmation must get the SAME request instance the ack was sent with");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A confirmation still pending when the actor stops is abandoned AND reported failed, so the host reverts the
|
||||
/// ack this incarnation can no longer confirm. (A confirmation continuation posted back to the stopped actor
|
||||
/// would dead-letter and never report.)
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_confirmation_pending_when_the_actor_stops_is_reported_as_a_failed_ack()
|
||||
{
|
||||
var driver = new ConfirmingDriver();
|
||||
var actor = SpawnConnected(driver);
|
||||
driver.Confirmation = () => new TaskCompletionSource().Task; // never completes on its own
|
||||
var router = CreateTestProbe();
|
||||
|
||||
actor.Tell(Ack("n-stop"), router.Ref);
|
||||
AwaitCondition(() => driver.ConfirmCalls == 1, Timeout, TestContext.Current.CancellationToken);
|
||||
Sys.Stop(actor);
|
||||
|
||||
var failed = router.ExpectMsg<DriverInstanceActor.AlarmAckFailed>(Timeout, cancellationToken: TestContext.Current.CancellationToken);
|
||||
failed.ConditionNodeId.ShouldBe("n-stop");
|
||||
failed.Reason.ShouldContain("stopped");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A confirmation that succeeds reports nothing. Positive control: a later ack whose confirmation fails is
|
||||
/// reported, and it is the FIRST report the router receives â so none came for the confirmed one.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_confirmed_ack_reports_nothing()
|
||||
{
|
||||
var driver = new ConfirmingDriver();
|
||||
var actor = SpawnConnected(driver);
|
||||
var router = CreateTestProbe();
|
||||
|
||||
// Already complete: the actor's confirmation task finishes (and would report) right after the call that
|
||||
// bumps ConfirmCalls, long before the second ack below makes its actor round trip.
|
||||
driver.Confirmation = () => Task.CompletedTask;
|
||||
actor.Tell(Ack("n-ok"), router.Ref);
|
||||
AwaitCondition(() => driver.ConfirmCalls == 1, Timeout, TestContext.Current.CancellationToken);
|
||||
|
||||
driver.Confirmation = () => Task.FromException(new DriverOperationRefusedException("unconfirmed"));
|
||||
actor.Tell(Ack("n-bad"), router.Ref);
|
||||
|
||||
router.ExpectMsg<DriverInstanceActor.AlarmAckFailed>(Timeout, cancellationToken: TestContext.Current.CancellationToken).ConditionNodeId.ShouldBe("n-bad");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 review — the failure echoes the routed ack's <see cref="DriverInstanceActor.RouteAlarmAck.AckSequence"/>,
|
||||
/// which is what lets the host revert only the acknowledge that failed (keyed in
|
||||
/// <c>NativeAlarmProjector.TryRevertAcknowledge(nodeId, ackSequence, …)</c>).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_failed_confirmation_echoes_the_ack_sequence()
|
||||
{
|
||||
var driver = new ConfirmingDriver();
|
||||
var actor = SpawnConnected(driver);
|
||||
driver.Confirmation = () => Task.FromException(new DriverOperationRefusedException("unconfirmed"));
|
||||
var router = CreateTestProbe();
|
||||
|
||||
actor.Tell(Ack("n-seq") with { AckSequence = 42 }, router.Ref);
|
||||
|
||||
router.ExpectMsg<DriverInstanceActor.AlarmAckFailed>(Timeout, cancellationToken: TestContext.Current.CancellationToken)
|
||||
.AckSequence.ShouldBe(42L);
|
||||
}
|
||||
|
||||
private static DriverInstanceActor.RouteAlarmAck Ack(string conditionNodeId) =>
|
||||
new("Tank.Hi", "c", "alice", ConditionNodeId: conditionNodeId);
|
||||
|
||||
/// <summary>Spawns the actor as a child of the test actor and waits for its Connected announcement.</summary>
|
||||
private IActorRef SpawnConnected(ConfirmingDriver driver)
|
||||
{
|
||||
var actor = ChildActorOf(DriverInstanceActor.Props(driver));
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
FishForMessage<DriverInstanceActor.ConnectivityChanged>(c => c.Connected, Timeout);
|
||||
return actor;
|
||||
}
|
||||
|
||||
private sealed class ConfirmingDriver : IDriver, IAlarmSource, IAlarmAcknowledgeConfirmation
|
||||
{
|
||||
private int _ackCalls;
|
||||
private int _confirmCalls;
|
||||
private readonly ConcurrentQueue<IReadOnlyList<AlarmAcknowledgeRequest>> _ackedLists = new();
|
||||
private int _foreignConfirmations;
|
||||
|
||||
/// <summary>Confirmations asked for a request list that was NOT the instance passed to AcknowledgeAsync —
|
||||
/// the contract a driver matching by instance (Galaxy) relies on.</summary>
|
||||
public int ForeignConfirmations => Volatile.Read(ref _foreignConfirmations);
|
||||
|
||||
public Func<Task>? Confirmation { get; set; }
|
||||
|
||||
public int AckCalls => Volatile.Read(ref _ackCalls);
|
||||
|
||||
public int ConfirmCalls => Volatile.Read(ref _confirmCalls);
|
||||
|
||||
public string DriverInstanceId => "drv-confirm";
|
||||
|
||||
public string DriverType => "Test";
|
||||
|
||||
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, null, null);
|
||||
|
||||
public Task<IAlarmSubscriptionHandle> SubscribeAlarmsAsync(
|
||||
IReadOnlyList<string> sourceNodeIds, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IAlarmSubscriptionHandle>(new Handle());
|
||||
|
||||
public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken) =>
|
||||
Task.CompletedTask;
|
||||
|
||||
public Task AcknowledgeAsync(
|
||||
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken)
|
||||
{
|
||||
_ackedLists.Enqueue(acknowledgements);
|
||||
Interlocked.Increment(ref _ackCalls);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task ConfirmAcknowledgedAsync(
|
||||
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!_ackedLists.Any(l => ReferenceEquals(l, acknowledgements))) Interlocked.Increment(ref _foreignConfirmations);
|
||||
Interlocked.Increment(ref _confirmCalls);
|
||||
return (Confirmation?.Invoke() ?? Task.CompletedTask).WaitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public event EventHandler<AlarmEventArgs>? OnAlarmEvent
|
||||
{
|
||||
add { }
|
||||
remove { }
|
||||
}
|
||||
|
||||
private sealed class Handle : IAlarmSubscriptionHandle
|
||||
{
|
||||
public string DiagnosticId => "h";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -606,6 +606,56 @@ public class NativeAlarmProjectorTests
|
||||
snapshot.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 — a transition that only CARRIES the optimistic ack through (a Clear: the alarm returned to
|
||||
/// normal while the ack was still being confirmed upstream) does not settle it: a late failure still reverts,
|
||||
/// to inactive + UNacked (Galaxy's UNACK_RTN). Positive control: an Acknowledge transition does settle it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_carry_through_transition_keeps_the_local_ack_pending()
|
||||
{
|
||||
var sut = new NativeAlarmProjector();
|
||||
sut.Project("n1", Evt(AlarmTransitionKind.Raise));
|
||||
sut.MarkAcknowledged("n1");
|
||||
sut.Project("n1", Evt(AlarmTransitionKind.Clear)).Acknowledged.ShouldBeTrue("the optimistic ack rides along");
|
||||
|
||||
sut.TryRevertAcknowledge("n1", out var reverted).ShouldBeTrue();
|
||||
reverted!.Active.ShouldBeFalse();
|
||||
reverted.Acknowledged.ShouldBeFalse();
|
||||
|
||||
sut.Project("n1", Evt(AlarmTransitionKind.Raise));
|
||||
sut.MarkAcknowledged("n1");
|
||||
sut.Project("n1", Evt(AlarmTransitionKind.Raise)).Acknowledged.ShouldBeTrue("a same-message Raise carries it too");
|
||||
sut.Project("n1", Evt(AlarmTransitionKind.Acknowledge));
|
||||
sut.TryRevertAcknowledge("n1", out _).ShouldBeFalse("an Acknowledge transition states the ack — authoritative");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 review — a pending ack can outlive its own delivery (a source that sends no Acknowledge back,
|
||||
/// across a Clear that only carried it), so a revert is keyed to the failed ack's sequence: a later re-ack that
|
||||
/// changed nothing (sequence 0) or a stale earlier ack's failure does not undo it. Positive control: the
|
||||
/// matching sequence does.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_revert_only_undoes_the_acknowledge_that_failed()
|
||||
{
|
||||
var sut = new NativeAlarmProjector();
|
||||
sut.Project("n1", Evt(AlarmTransitionKind.Raise));
|
||||
var first = sut.MarkAcknowledged("n1");
|
||||
first.ShouldBeGreaterThan(0);
|
||||
sut.Project("n1", Evt(AlarmTransitionKind.Clear)); // first ack delivered; nothing settles it
|
||||
|
||||
var reAck = sut.MarkAcknowledged("n1"); // a re-ack of the acked, cleared condition
|
||||
reAck.ShouldBe(0L, "it changed nothing, so its failure has nothing to undo");
|
||||
sut.TryRevertAcknowledge("n1", reAck, out _).ShouldBeFalse("a later re-ack's failure must not revert the first");
|
||||
|
||||
sut.Project("n1", Evt(AlarmTransitionKind.Raise, msg: "again")); // a new occurrence (un-acks)
|
||||
var second = sut.MarkAcknowledged("n1");
|
||||
sut.TryRevertAcknowledge("n1", first, out _).ShouldBeFalse("a stale earlier ack's failure must not revert the newer one");
|
||||
sut.TryRevertAcknowledge("n1", second, out var reverted).ShouldBeTrue();
|
||||
reverted!.Acknowledged.ShouldBeFalse();
|
||||
}
|
||||
|
||||
private static AlarmEventArgs Evt(
|
||||
AlarmTransitionKind kind,
|
||||
AlarmSeverity sev = AlarmSeverity.High,
|
||||
|
||||
Reference in New Issue
Block a user