fix(galaxy): confirm unconfirmed native alarm acks against Galaxy's acknowledged transition (#869)
mxaccessgw answers AcknowledgeAlarm OK both when the AckMsg write was applied and when it was only sent (no OnWriteComplete; the reply then carries a diagnostic). EvaluateReply treated both as success, so the H6 live gate saw a condition go Acknowledged while Galaxy never applied it. - EvaluateReply returns Applied (no diagnostic) or Unconfirmed (diagnostic) and logs the gateway diagnostic on every branch that carries one. - GalaxyAckConfirmationTracker watches the alarm feed from before the RPC and, for an unconfirmed reply, waits up to AlarmAck.ConfirmTimeoutMs (default 10 s, 0 = old behaviour) for an Acknowledge/ActiveAcknowledged transition; on timeout it refuses (BadTimeout + diagnostic). - New optional IAlarmAcknowledgeConfirmation: DriverInstanceActor awaits it after AcknowledgeAsync, off its mailbox and outside the resilience pipeline; a failure is the existing #760 AlarmAckFailed revert. - AdminUI Galaxy form carries the new knob; docs updated. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -317,10 +317,10 @@ public sealed class GalaxyDriverAlarmSourceTests
|
||||
/// <param name="credential">The resolved Galaxy credential.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A completed task.</returns>
|
||||
public Task AcknowledgeAsync(string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
public Task<GalaxyAckOutcome> AcknowledgeAsync(string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
{
|
||||
Calls.Add((alarmFullReference, comment, credential.User));
|
||||
return Task.CompletedTask;
|
||||
return Task.FromResult(GalaxyAckOutcome.Applied);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+381
@@ -0,0 +1,381 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 — mxaccessgw answers <c>AcknowledgeAlarm</c> OK both when the <c>AckMsg</c> write was APPLIED and
|
||||
/// when it was only SENT (no <c>OnWriteComplete</c> inside the worker's window; the reply then carries a
|
||||
/// diagnostic). The H6 live gate saw an unconfirmed ack reported Acknowledged while Galaxy never applied it.
|
||||
/// The driver now treats an unconfirmed OK as pending and confirms it against Galaxy's own acknowledged
|
||||
/// transition on the alarm feed, bounded by <c>AlarmAck.ConfirmTimeoutMs</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The "fake gateway" is a scripted <see cref="AcknowledgeAlarmReply"/> run through the production
|
||||
/// <see cref="GatewayGalaxyAlarmAcknowledger.EvaluateReply"/> — the gRPC transport seam is internal to the
|
||||
/// client package, so the reply mapping is the part under test.
|
||||
/// </remarks>
|
||||
public sealed class GalaxyAckConfirmationTests
|
||||
{
|
||||
private const string AlarmRef = "Tank01.Level.HiHi";
|
||||
|
||||
private const string UnconfirmedDiagnostic =
|
||||
"AckMsg write to 'Tank01.Level.HiHi.AckMsg' sent; OnWriteComplete not observed within 5000 ms.";
|
||||
|
||||
private static readonly GalaxyAckCredential Alice =
|
||||
new("alice", "CORP", "alice-pw", GalaxyAckCredentialSource.ApiKey);
|
||||
|
||||
// ------------------------------------------------------------------ reply mapping
|
||||
|
||||
[Fact]
|
||||
public void EvaluateReply_ok_without_diagnostic_is_applied()
|
||||
{
|
||||
var logger = new CapturingLogger();
|
||||
|
||||
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(AppliedReply(), AlarmRef, Alice, logger);
|
||||
|
||||
outcome.IsApplied.ShouldBeTrue();
|
||||
logger.Entries.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
/// <summary>The H6 reply: OK + hresult 0 + a "sent; not observed" diagnostic is UNCONFIRMED, and the
|
||||
/// diagnostic is logged. Before #869 the same reply returned silently as success.</summary>
|
||||
[Fact]
|
||||
public void EvaluateReply_ok_with_diagnostic_is_unconfirmed_and_logs_the_diagnostic()
|
||||
{
|
||||
var logger = new CapturingLogger();
|
||||
|
||||
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(UnconfirmedReply(), AlarmRef, Alice, logger);
|
||||
|
||||
outcome.IsApplied.ShouldBeFalse();
|
||||
outcome.Diagnostic.ShouldBe(UnconfirmedDiagnostic);
|
||||
logger.AllText.ShouldContain(UnconfirmedDiagnostic);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateReply_native_refusal_logs_the_diagnostic_and_refuses()
|
||||
{
|
||||
var reply = new AcknowledgeAlarmReply
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
Hresult = 1008,
|
||||
DiagnosticMessage = "AckMsg write to 'Tank01.Level.HiHi.AckMsg' rejected: category SecurityError, detail 1008.",
|
||||
};
|
||||
var logger = new CapturingLogger();
|
||||
|
||||
Should.Throw<DriverOperationRefusedException>(
|
||||
() => GatewayGalaxyAlarmAcknowledger.EvaluateReply(reply, AlarmRef, Alice, logger));
|
||||
|
||||
logger.AllText.ShouldContain("rejected: category SecurityError");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EvaluateReply_scrubs_the_password_from_the_diagnostic()
|
||||
{
|
||||
var reply = new AcknowledgeAlarmReply
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
Hresult = 0,
|
||||
DiagnosticMessage = "sent as alice/alice-pw; not observed",
|
||||
};
|
||||
var logger = new CapturingLogger();
|
||||
|
||||
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(reply, AlarmRef, Alice, logger);
|
||||
|
||||
outcome.Diagnostic.ShouldNotContain("alice-pw");
|
||||
logger.AllText.ShouldNotContain("alice-pw");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ driver: confirmation
|
||||
|
||||
/// <summary>Applied OK → acknowledged: nothing pending, the confirmation completes at once.</summary>
|
||||
[Fact]
|
||||
public async Task Applied_ack_is_confirmed_at_once()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(AppliedReply());
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 30_000);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
||||
|
||||
confirm.IsCompletedSuccessfully.ShouldBeTrue("an applied ack needs no confirmation");
|
||||
gateway.Calls.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unconfirmed OK + Galaxy's acknowledged transition on the feed → acknowledged. Positive control: the
|
||||
/// confirmation is still pending after transitions that must NOT confirm (another alarm's ack, this
|
||||
/// alarm's raise) — it only completes on the matching one.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Unconfirmed_ack_is_confirmed_by_the_acknowledged_transition()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(UnconfirmedReply());
|
||||
var logger = new CapturingLogger();
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 120_000, logger);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
||||
|
||||
feed.Emit(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
|
||||
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Raise));
|
||||
feed.Emit(Transition(AlarmRef.ToUpperInvariant(), GalaxyAlarmTransitionKind.Acknowledge));
|
||||
|
||||
await TestWait.ForAsync(confirm, "the acknowledged transition must confirm the pending ack");
|
||||
logger.AllText.ShouldContain(UnconfirmedDiagnostic);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The feed and the unary reply race: Galaxy's acknowledged transition can arrive BEFORE the RPC answers
|
||||
/// "unconfirmed". The watch is registered before the RPC, so that early transition still confirms — with a
|
||||
/// 1 ms window, a lost transition would time out instead.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Acknowledged_transition_that_overtakes_the_reply_still_confirms()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(UnconfirmedReply())
|
||||
{
|
||||
DuringCall = () => feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Acknowledge)),
|
||||
};
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 1);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
await driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unconfirmed OK + no acknowledged transition → the confirmation fails with a refusal naming the timeout and
|
||||
/// the gateway diagnostic (the runtime reverts the condition on it), and a Warning carrying the diagnostic is
|
||||
/// logged. Transitions that must not confirm (another alarm's ack, this alarm's retrigger) are emitted during
|
||||
/// the wait. Before #869 this ack counted as acknowledged.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Unconfirmed_ack_without_confirmation_fails_with_the_diagnostic()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(UnconfirmedReply());
|
||||
var logger = new CapturingLogger();
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 300, logger);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
||||
feed.Emit(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
|
||||
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Retrigger));
|
||||
|
||||
var ex = await Should.ThrowAsync<DriverOperationRefusedException>(() => confirm);
|
||||
ex.Message.ShouldContain(AlarmRef);
|
||||
ex.Message.ShouldContain("BadTimeout");
|
||||
ex.Message.ShouldContain(UnconfirmedDiagnostic);
|
||||
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Text.Contains(UnconfirmedDiagnostic));
|
||||
}
|
||||
|
||||
/// <summary>A refused ack (non-zero native code) throws from AcknowledgeAsync — the existing revert path —
|
||||
/// and leaves nothing pending to confirm.</summary>
|
||||
[Fact]
|
||||
public async Task Refused_ack_throws_and_leaves_nothing_pending()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(new AcknowledgeAlarmReply
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
Hresult = unchecked((int)0x80070005),
|
||||
});
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 30_000);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await Should.ThrowAsync<DriverOperationRefusedException>(
|
||||
() => driver.AcknowledgeAsync(request, CancellationToken.None));
|
||||
|
||||
driver.ConfirmAcknowledgedAsync(request, CancellationToken.None).IsCompletedSuccessfully.ShouldBeTrue();
|
||||
driver.PendingAckConfirmations.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary><c>ConfirmTimeoutMs = 0</c> is the opt-out: an unconfirmed ack counts as acknowledged, with a Warning.</summary>
|
||||
[Fact]
|
||||
public async Task Zero_timeout_counts_an_unconfirmed_ack_as_acknowledged_with_a_warning()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(UnconfirmedReply());
|
||||
var logger = new CapturingLogger();
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 0, logger);
|
||||
var request = Ack(AlarmRef);
|
||||
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
|
||||
driver.ConfirmAcknowledgedAsync(request, CancellationToken.None).IsCompletedSuccessfully.ShouldBeTrue();
|
||||
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Text.Contains("confirmation is disabled"));
|
||||
driver.PendingAckConfirmations.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>Each confirmation is for ITS request instance: confirming one leaves no watch behind, and a later
|
||||
/// ack of the same alarm is confirmed independently.</summary>
|
||||
[Fact]
|
||||
public async Task Confirmations_release_their_watches()
|
||||
{
|
||||
var feed = new FakeAlarmFeed();
|
||||
var gateway = new ScriptedGateway(UnconfirmedReply());
|
||||
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 120_000);
|
||||
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
||||
|
||||
for (var i = 0; i < 2; i++)
|
||||
{
|
||||
var request = Ack(AlarmRef);
|
||||
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
||||
driver.PendingAckConfirmations.ShouldBe(1);
|
||||
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
||||
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.ActiveAcknowledged));
|
||||
await TestWait.ForAsync(confirm, $"ack #{i} must be confirmed");
|
||||
driver.PendingAckConfirmations.ShouldBe(0);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Options_default_to_ten_seconds_and_map_negative_to_the_default()
|
||||
{
|
||||
new GalaxyAlarmAckOptions().ConfirmTimeout.ShouldBe(TimeSpan.FromSeconds(10));
|
||||
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = -5 }.ConfirmTimeout.ShouldBe(TimeSpan.FromSeconds(10));
|
||||
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 0 }.ConfirmTimeout.ShouldBe(TimeSpan.Zero);
|
||||
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 999_999 }.ConfirmTimeout.ShouldBe(TimeSpan.FromMinutes(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Factory_binds_ConfirmTimeoutMs_and_defaults_when_absent()
|
||||
{
|
||||
const string Json = """
|
||||
{ "gateway": { "endpoint": "https://gw:5001", "apiKeySecretRef": "dev:k" },
|
||||
"mxAccess": { "clientName": "C" }, "alarmAck": { "confirmTimeoutMs": 2500 } }
|
||||
""";
|
||||
using var withKnob = GalaxyDriverFactoryExtensions.CreateInstance("g", Json);
|
||||
withKnob.AckConfirmationWindow.ShouldBe(TimeSpan.FromMilliseconds(2500));
|
||||
|
||||
using var without = GalaxyDriverFactoryExtensions.CreateInstance(
|
||||
"g", """{ "gateway": { "endpoint": "https://gw:5001", "apiKeySecretRef": "dev:k" }, "mxAccess": { "clientName": "C" } }""");
|
||||
without.AckConfirmationWindow.ShouldBe(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ helpers
|
||||
|
||||
private static AcknowledgeAlarmReply AppliedReply() => new()
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
Hresult = 0,
|
||||
};
|
||||
|
||||
private static AcknowledgeAlarmReply UnconfirmedReply() => new()
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
Hresult = 0,
|
||||
DiagnosticMessage = UnconfirmedDiagnostic,
|
||||
};
|
||||
|
||||
private static AlarmAcknowledgeRequest[] Ack(string conditionId) =>
|
||||
[new AlarmAcknowledgeRequest(SourceNodeId: "Tank01", ConditionId: conditionId, Comment: "c")];
|
||||
|
||||
private static GalaxyDriver NewDriver(
|
||||
IGalaxyAlarmFeed feed, IGalaxyAlarmAcknowledger acknowledger, int confirmTimeoutMs, CapturingLogger? logger = null)
|
||||
{
|
||||
var options = new GalaxyDriverOptions(
|
||||
new GalaxyGatewayOptions("http://localhost:5000", "literal-api-key"),
|
||||
new GalaxyMxAccessOptions("AckConfirmTest"),
|
||||
new GalaxyRepositoryOptions(),
|
||||
new GalaxyReconnectOptions())
|
||||
{
|
||||
AlarmAck = new GalaxyAlarmAckOptions { ConfirmTimeoutMs = confirmTimeoutMs },
|
||||
};
|
||||
return new GalaxyDriver(
|
||||
driverInstanceId: "drv-ack",
|
||||
options: options,
|
||||
hierarchySource: null,
|
||||
alarmAcknowledger: acknowledger,
|
||||
alarmFeed: feed,
|
||||
logger: logger is null ? null : new TypedLogger<GalaxyDriver>(logger),
|
||||
ackCredentialResolver: FixedAckCredentialResolver.Fallback());
|
||||
}
|
||||
|
||||
private static GalaxyAlarmTransition Transition(string alarmRef, GalaxyAlarmTransitionKind kind) => new(
|
||||
AlarmFullReference: alarmRef,
|
||||
SourceObjectReference: "Tank01",
|
||||
AlarmTypeName: "AnalogLimitAlarm.HiHi",
|
||||
TransitionKind: kind,
|
||||
SeverityBucket: AlarmSeverity.High,
|
||||
OpcUaSeverity: 700,
|
||||
RawMxAccessSeverity: 500,
|
||||
OriginalRaiseTimestampUtc: null,
|
||||
TransitionTimestampUtc: DateTime.UtcNow,
|
||||
OperatorUser: string.Empty,
|
||||
OperatorComment: string.Empty,
|
||||
Category: "Process",
|
||||
Description: "Tank 01 high-high level");
|
||||
|
||||
/// <summary>
|
||||
/// A fake gateway: every call answers with <paramref name="reply"/>, mapped by the production
|
||||
/// <see cref="GatewayGalaxyAlarmAcknowledger.EvaluateReply"/>. <see cref="DuringCall"/> runs while the RPC is
|
||||
/// "in flight" (before the reply is evaluated).
|
||||
/// </summary>
|
||||
private sealed class ScriptedGateway(AcknowledgeAlarmReply reply) : IGalaxyAlarmAcknowledger
|
||||
{
|
||||
private int _calls;
|
||||
|
||||
public Action? DuringCall { get; init; }
|
||||
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
public Task<GalaxyAckOutcome> AcknowledgeAsync(
|
||||
string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _calls);
|
||||
DuringCall?.Invoke();
|
||||
try
|
||||
{
|
||||
return Task.FromResult(GatewayGalaxyAlarmAcknowledger.EvaluateReply(
|
||||
reply, alarmFullReference, credential, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return Task.FromException<GalaxyAckOutcome>(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeAlarmFeed : IGalaxyAlarmFeed
|
||||
{
|
||||
public event EventHandler<GalaxyAlarmTransition>? OnAlarmTransition;
|
||||
|
||||
public void Start() { }
|
||||
|
||||
public void Emit(GalaxyAlarmTransition transition) => OnAlarmTransition?.Invoke(this, transition);
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class TypedLogger<T>(CapturingLogger inner) : ILogger<T>
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(LogLevel l, EventId id, TState s, Exception? e, Func<TState, Exception?, string> f)
|
||||
{
|
||||
lock (_gate) inner.Log(l, id, s, e, f);
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -485,11 +485,11 @@ public sealed class GalaxyAckCredentialTests
|
||||
{
|
||||
public List<(string AlarmRef, string Comment, GalaxyAckCredential Credential)> Calls { get; } = [];
|
||||
|
||||
public Task AcknowledgeAsync(
|
||||
public Task<GalaxyAckOutcome> AcknowledgeAsync(
|
||||
string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
{
|
||||
Calls.Add((alarmFullReference, comment, credential));
|
||||
return Task.CompletedTask;
|
||||
return Task.FromResult(GalaxyAckOutcome.Applied);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -242,10 +242,10 @@ public sealed class GalaxyDriverRawPathKeyingTests
|
||||
{
|
||||
public List<(string AlarmRef, string Comment, string Operator)> Calls { get; } = [];
|
||||
|
||||
public Task AcknowledgeAsync(string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
public Task<GalaxyAckOutcome> AcknowledgeAsync(string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
||||
{
|
||||
Calls.Add((alarmFullReference, comment, credential.User));
|
||||
return Task.CompletedTask;
|
||||
return Task.FromResult(GalaxyAckOutcome.Applied);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+8
-8
@@ -34,22 +34,22 @@ public sealed class GatewayGalaxyAlarmAcknowledgerTests
|
||||
/// </summary>
|
||||
private sealed class FakeAcknowledger : IGalaxyAlarmAcknowledger
|
||||
{
|
||||
private readonly Queue<Func<Task>> _outcomes = new();
|
||||
private readonly Queue<Func<Task<GalaxyAckOutcome>>> _outcomes = new();
|
||||
|
||||
/// <summary>Calls recorded in the order received.</summary>
|
||||
public List<(string AlarmRef, string Comment, string Operator)> Calls { get; } = [];
|
||||
|
||||
/// <summary>Enqueues an outcome for the next call (completes normally or throws).</summary>
|
||||
public void Enqueue(Func<Task> outcome) => _outcomes.Enqueue(outcome);
|
||||
public void Enqueue(Func<Task<GalaxyAckOutcome>> outcome) => _outcomes.Enqueue(outcome);
|
||||
|
||||
public Task AcknowledgeAsync(
|
||||
public Task<GalaxyAckOutcome> AcknowledgeAsync(
|
||||
string alarmFullReference,
|
||||
string comment,
|
||||
GalaxyAckCredential credential,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Calls.Add((alarmFullReference, comment, credential.User));
|
||||
return _outcomes.Count > 0 ? _outcomes.Dequeue()() : Task.CompletedTask;
|
||||
return _outcomes.Count > 0 ? _outcomes.Dequeue()() : Task.FromResult(GalaxyAckOutcome.Applied);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,9 +77,9 @@ public sealed class GatewayGalaxyAlarmAcknowledgerTests
|
||||
{
|
||||
var fake = new FakeAcknowledger();
|
||||
// First call: simulates a transient transport blip (e.g. RpcException from gRPC).
|
||||
fake.Enqueue(() => Task.FromException(new IOException("synthetic transport fault")));
|
||||
fake.Enqueue(() => Task.FromException<GalaxyAckOutcome>(new IOException("synthetic transport fault")));
|
||||
// Second call: succeeds — proves no dead-state latch.
|
||||
fake.Enqueue(() => Task.CompletedTask);
|
||||
fake.Enqueue(() => Task.FromResult(GalaxyAckOutcome.Applied));
|
||||
|
||||
var driver = new GalaxyDriver("g", Opts(),
|
||||
hierarchySource: null, alarmAcknowledger: fake,
|
||||
@@ -109,8 +109,8 @@ public sealed class GatewayGalaxyAlarmAcknowledgerTests
|
||||
public async Task Acknowledge_after_protocol_failure_still_issues_next_call()
|
||||
{
|
||||
var fake = new FakeAcknowledger();
|
||||
fake.Enqueue(() => Task.FromException(new InvalidOperationException("gateway overload")));
|
||||
fake.Enqueue(() => Task.CompletedTask);
|
||||
fake.Enqueue(() => Task.FromException<GalaxyAckOutcome>(new InvalidOperationException("gateway overload")));
|
||||
fake.Enqueue(() => Task.FromResult(GalaxyAckOutcome.Applied));
|
||||
|
||||
var driver = new GalaxyDriver("g", Opts(),
|
||||
hierarchySource: null, alarmAcknowledger: fake,
|
||||
|
||||
+26
@@ -259,4 +259,30 @@ public sealed class GalaxyDriverPageFormSerializationTests
|
||||
json.ShouldNotContain("hasFallback");
|
||||
JsonSerializer.Deserialize<GalaxyDriverOptions>(json, _pageOpts)!.AlarmAck.ShouldBe(original.AlarmAck);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869: the ack-confirmation window must survive FromRecord/ToRecord and JSON, or saving the page
|
||||
/// would silently reset an operator's setting (e.g. the 0 = do-not-wait opt-out) to the default.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void FormModel_RoundTrips_AlarmAckConfirmTimeout()
|
||||
{
|
||||
var original = new GalaxyDriverOptions(
|
||||
new GalaxyGatewayOptions("https://gw:5001", "env:K"),
|
||||
new GalaxyMxAccessOptions("OtOpcUa"),
|
||||
new GalaxyRepositoryOptions(),
|
||||
new GalaxyReconnectOptions())
|
||||
{
|
||||
AlarmAck = new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 0 },
|
||||
};
|
||||
|
||||
var back = GalaxyDriverForm.GalaxyFormModel.FromRecord(original).ToRecord();
|
||||
back.AlarmAck.ConfirmTimeoutMs.ShouldBe(0);
|
||||
|
||||
var json = JsonSerializer.Serialize(back, _pageOpts);
|
||||
json.ShouldContain("\"confirmTimeoutMs\":0");
|
||||
json.ShouldNotContain("\"confirmTimeout\":");
|
||||
JsonSerializer.Deserialize<GalaxyDriverOptions>(json, _pageOpts)!.AlarmAck.ConfirmTimeoutMs.ShouldBe(0);
|
||||
new GalaxyDriverForm.GalaxyFormModel().AckConfirmTimeoutMs.ShouldBe(GalaxyAlarmAckOptions.DefaultConfirmTimeoutMs);
|
||||
}
|
||||
}
|
||||
|
||||
+82
-1
@@ -217,6 +217,9 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
RaiseAndExpectUnacked(actor, publish);
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
|
||||
AwaitAssert(() => recorder.Acks.Count.ShouldBe(1), duration: Timeout, cancellationToken: TestContext.Current.CancellationToken);
|
||||
// lmxopcua#869: the driver is an IAlarmAcknowledgeConfirmation whose confirmation succeeds at once — it was
|
||||
// asked, and a confirmed ack must not revert.
|
||||
AwaitAssert(() => recorder.ConfirmCalls.ShouldBe(1), duration: Timeout, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
actor.Tell(new DriverInstanceActor.AttributeAlarmPublished("drv-1", Evt(AlarmTransitionKind.Clear)));
|
||||
var next = NextAlarmUpdate(publish);
|
||||
@@ -224,6 +227,62 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
next.State.Acknowledged.ShouldBeTrue("the successful ack must have survived — no revert");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 — the driver ACCEPTED the ack (AcknowledgeAsync returned normally, e.g. the gateway's "sent but
|
||||
/// unconfirmed" OK) but its upstream never confirmed it: the confirmation refusal reverts the condition to
|
||||
/// unacknowledged exactly like an AcknowledgeAsync failure. Before #869 nothing awaited a confirmation, so the
|
||||
/// condition stayed Acknowledged although Galaxy never applied the ack.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void An_accepted_but_unconfirmed_ack_reverts_the_condition_to_unacknowledged()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway")
|
||||
{
|
||||
Confirmation = () => Task.FromException(new DriverOperationRefusedException(
|
||||
"Galaxy did not confirm the acknowledge of 'Tank.Hi' within 10 s (BadTimeout; gateway: AckMsg write sent; OnWriteComplete not observed)")),
|
||||
};
|
||||
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
|
||||
var publish = this.CreateCleanOutcomePublishProbe();
|
||||
var actor = SpawnHostAndApply(db, deploymentId, recorder, publish.Ref);
|
||||
|
||||
RaiseAndExpectUnacked(actor, publish);
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
|
||||
|
||||
var reverted = NextAlarmUpdate(publish);
|
||||
reverted.AlarmNodeId.ShouldBe(AlarmRawPath);
|
||||
reverted.State.Active.ShouldBeTrue();
|
||||
reverted.State.Acknowledged.ShouldBeFalse("an ack the upstream never confirmed must not stand");
|
||||
recorder.Acks.Count.ShouldBe(1);
|
||||
recorder.ConfirmCalls.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#869 — the confirmation wait runs OFF the driver actor's mailbox: while one ack's confirmation is
|
||||
/// still pending (it never completes here), a second ack is still routed to the driver. A confirmation awaited
|
||||
/// inside the ack handler would suspend the mailbox and the second ack would never arrive.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_pending_confirmation_does_not_block_the_driver_actor()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var never = new TaskCompletionSource();
|
||||
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway")
|
||||
{
|
||||
Confirmation = () => never.Task,
|
||||
};
|
||||
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
|
||||
var actor = SpawnHostAndApply(db, deploymentId, recorder);
|
||||
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "first", "alice"));
|
||||
AwaitAssert(() => recorder.ConfirmCalls.ShouldBe(1), duration: Timeout, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "second", "alice"));
|
||||
AwaitAssert(() => recorder.Acks.ShouldContain(a => a.Comment == "second"), duration: Timeout,
|
||||
cancellationToken: TestContext.Current.CancellationToken);
|
||||
never.TrySetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#760 — the host publishes the condition → owning-driver pre-check snapshot the OPC UA node
|
||||
/// manager uses to refuse an acknowledge synchronously: the driver's own verdict for the condition it
|
||||
@@ -452,6 +511,12 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
/// <summary>The driver's synchronous pre-check verdict (lmxopcua#760); null ⇒ always proceed.</summary>
|
||||
public Func<string?, string?>? PreflightRefusal { init => _driver.PreflightRefusal = value; }
|
||||
|
||||
/// <summary>lmxopcua#869: what <see cref="IAlarmAcknowledgeConfirmation.ConfirmAcknowledgedAsync"/> returns; null ⇒ confirmed at once.</summary>
|
||||
public Func<Task>? Confirmation { init => _driver.Confirmation = value; }
|
||||
|
||||
/// <summary>How many confirmations the spawned driver was asked for.</summary>
|
||||
public int ConfirmCalls => _driver.ConfirmCalls;
|
||||
|
||||
/// <inheritdoc />
|
||||
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
|
||||
{
|
||||
@@ -465,8 +530,24 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
}
|
||||
|
||||
/// <summary>An <see cref="IDriver"/> + <see cref="IAlarmSource"/> that records every acknowledge.</summary>
|
||||
private sealed class RecordingAlarmDriver : IDriver, IAlarmSource, IAlarmAcknowledgePreflight
|
||||
private sealed class RecordingAlarmDriver : IDriver, IAlarmSource, IAlarmAcknowledgePreflight, IAlarmAcknowledgeConfirmation
|
||||
{
|
||||
/// <summary>lmxopcua#869: the confirmation outcome; null ⇒ confirmed at once.</summary>
|
||||
public Func<Task>? Confirmation { get; set; }
|
||||
|
||||
private int _confirmCalls;
|
||||
|
||||
/// <summary>How many confirmations were requested.</summary>
|
||||
public int ConfirmCalls => Volatile.Read(ref _confirmCalls);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ConfirmAcknowledgedAsync(
|
||||
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _confirmCalls);
|
||||
return Confirmation?.Invoke() ?? Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>When set, <see cref="AcknowledgeAsync"/> records the request and then throws this.</summary>
|
||||
public Exception? AckFault { get; set; }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user