fix(galaxy): confirm unconfirmed native alarm acks against Galaxy's acknowledged transition (#869)

mxaccessgw answers AcknowledgeAlarm OK both when the AckMsg write was
applied and when it was only sent (no OnWriteComplete; the reply then
carries a diagnostic). EvaluateReply treated both as success, so the H6
live gate saw a condition go Acknowledged while Galaxy never applied it.

- EvaluateReply returns Applied (no diagnostic) or Unconfirmed (diagnostic)
  and logs the gateway diagnostic on every branch that carries one.
- GalaxyAckConfirmationTracker watches the alarm feed from before the RPC
  and, for an unconfirmed reply, waits up to AlarmAck.ConfirmTimeoutMs
  (default 10 s, 0 = old behaviour) for an Acknowledge/ActiveAcknowledged
  transition; on timeout it refuses (BadTimeout + diagnostic).
- New optional IAlarmAcknowledgeConfirmation: DriverInstanceActor awaits it
  after AcknowledgeAsync, off its mailbox and outside the resilience
  pipeline; a failure is the existing #760 AlarmAckFailed revert.
- AdminUI Galaxy form carries the new knob; docs updated.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:03:04 -05:00
parent e857135e8b
commit 4e492e6550
19 changed files with 1046 additions and 34 deletions
@@ -259,4 +259,30 @@ public sealed class GalaxyDriverPageFormSerializationTests
json.ShouldNotContain("hasFallback");
JsonSerializer.Deserialize<GalaxyDriverOptions>(json, _pageOpts)!.AlarmAck.ShouldBe(original.AlarmAck);
}
/// <summary>
/// lmxopcua#869: the ack-confirmation window must survive FromRecord/ToRecord and JSON, or saving the page
/// would silently reset an operator's setting (e.g. the 0 = do-not-wait opt-out) to the default.
/// </summary>
[Fact]
public void FormModel_RoundTrips_AlarmAckConfirmTimeout()
{
var original = new GalaxyDriverOptions(
new GalaxyGatewayOptions("https://gw:5001", "env:K"),
new GalaxyMxAccessOptions("OtOpcUa"),
new GalaxyRepositoryOptions(),
new GalaxyReconnectOptions())
{
AlarmAck = new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 0 },
};
var back = GalaxyDriverForm.GalaxyFormModel.FromRecord(original).ToRecord();
back.AlarmAck.ConfirmTimeoutMs.ShouldBe(0);
var json = JsonSerializer.Serialize(back, _pageOpts);
json.ShouldContain("\"confirmTimeoutMs\":0");
json.ShouldNotContain("\"confirmTimeout\":");
JsonSerializer.Deserialize<GalaxyDriverOptions>(json, _pageOpts)!.AlarmAck.ConfirmTimeoutMs.ShouldBe(0);
new GalaxyDriverForm.GalaxyFormModel().AckConfirmTimeoutMs.ShouldBe(GalaxyAlarmAckOptions.DefaultConfirmTimeoutMs);
}
}
@@ -217,6 +217,9 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
RaiseAndExpectUnacked(actor, publish);
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
AwaitAssert(() => recorder.Acks.Count.ShouldBe(1), duration: Timeout, cancellationToken: TestContext.Current.CancellationToken);
// lmxopcua#869: the driver is an IAlarmAcknowledgeConfirmation whose confirmation succeeds at once — it was
// asked, and a confirmed ack must not revert.
AwaitAssert(() => recorder.ConfirmCalls.ShouldBe(1), duration: Timeout, cancellationToken: TestContext.Current.CancellationToken);
actor.Tell(new DriverInstanceActor.AttributeAlarmPublished("drv-1", Evt(AlarmTransitionKind.Clear)));
var next = NextAlarmUpdate(publish);
@@ -224,6 +227,62 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
next.State.Acknowledged.ShouldBeTrue("the successful ack must have survived — no revert");
}
/// <summary>
/// lmxopcua#869 — the driver ACCEPTED the ack (AcknowledgeAsync returned normally, e.g. the gateway's "sent but
/// unconfirmed" OK) but its upstream never confirmed it: the confirmation refusal reverts the condition to
/// unacknowledged exactly like an AcknowledgeAsync failure. Before #869 nothing awaited a confirmation, so the
/// condition stayed Acknowledged although Galaxy never applied the ack.
/// </summary>
[Fact]
public void An_accepted_but_unconfirmed_ack_reverts_the_condition_to_unacknowledged()
{
var db = NewInMemoryDbFactory();
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway")
{
Confirmation = () => Task.FromException(new DriverOperationRefusedException(
"Galaxy did not confirm the acknowledge of 'Tank.Hi' within 10 s (BadTimeout; gateway: AckMsg write sent; OnWriteComplete not observed)")),
};
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
var publish = this.CreateCleanOutcomePublishProbe();
var actor = SpawnHostAndApply(db, deploymentId, recorder, publish.Ref);
RaiseAndExpectUnacked(actor, publish);
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
var reverted = NextAlarmUpdate(publish);
reverted.AlarmNodeId.ShouldBe(AlarmRawPath);
reverted.State.Active.ShouldBeTrue();
reverted.State.Acknowledged.ShouldBeFalse("an ack the upstream never confirmed must not stand");
recorder.Acks.Count.ShouldBe(1);
recorder.ConfirmCalls.ShouldBe(1);
}
/// <summary>
/// lmxopcua#869 — the confirmation wait runs OFF the driver actor's mailbox: while one ack's confirmation is
/// still pending (it never completes here), a second ack is still routed to the driver. A confirmation awaited
/// inside the ack handler would suspend the mailbox and the second ack would never arrive.
/// </summary>
[Fact]
public void A_pending_confirmation_does_not_block_the_driver_actor()
{
var db = NewInMemoryDbFactory();
var never = new TaskCompletionSource();
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway")
{
Confirmation = () => never.Task,
};
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
var actor = SpawnHostAndApply(db, deploymentId, recorder);
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "first", "alice"));
AwaitAssert(() => recorder.ConfirmCalls.ShouldBe(1), duration: Timeout, cancellationToken: TestContext.Current.CancellationToken);
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "second", "alice"));
AwaitAssert(() => recorder.Acks.ShouldContain(a => a.Comment == "second"), duration: Timeout,
cancellationToken: TestContext.Current.CancellationToken);
never.TrySetResult();
}
/// <summary>
/// lmxopcua#760 — the host publishes the condition → owning-driver pre-check snapshot the OPC UA node
/// manager uses to refuse an acknowledge synchronously: the driver's own verdict for the condition it
@@ -452,6 +511,12 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
/// <summary>The driver's synchronous pre-check verdict (lmxopcua#760); null ⇒ always proceed.</summary>
public Func<string?, string?>? PreflightRefusal { init => _driver.PreflightRefusal = value; }
/// <summary>lmxopcua#869: what <see cref="IAlarmAcknowledgeConfirmation.ConfirmAcknowledgedAsync"/> returns; null ⇒ confirmed at once.</summary>
public Func<Task>? Confirmation { init => _driver.Confirmation = value; }
/// <summary>How many confirmations the spawned driver was asked for.</summary>
public int ConfirmCalls => _driver.ConfirmCalls;
/// <inheritdoc />
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
{
@@ -465,8 +530,24 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
}
/// <summary>An <see cref="IDriver"/> + <see cref="IAlarmSource"/> that records every acknowledge.</summary>
private sealed class RecordingAlarmDriver : IDriver, IAlarmSource, IAlarmAcknowledgePreflight
private sealed class RecordingAlarmDriver : IDriver, IAlarmSource, IAlarmAcknowledgePreflight, IAlarmAcknowledgeConfirmation
{
/// <summary>lmxopcua#869: the confirmation outcome; null ⇒ confirmed at once.</summary>
public Func<Task>? Confirmation { get; set; }
private int _confirmCalls;
/// <summary>How many confirmations were requested.</summary>
public int ConfirmCalls => Volatile.Read(ref _confirmCalls);
/// <inheritdoc />
public Task ConfirmAcknowledgedAsync(
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken)
{
Interlocked.Increment(ref _confirmCalls);
return Confirmation?.Invoke() ?? Task.CompletedTask;
}
/// <summary>When set, <see cref="AcknowledgeAsync"/> records the request and then throws this.</summary>
public Exception? AckFault { get; set; }