Files
lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyAckConfirmationTests.cs
T
Joseph Doherty 77872b1d1c fix(runtime,galaxy): #869 review round 1 — confirmation off the actor scheduler, stop/async-fault tests
- Start the ack confirmation with Task.Run + ConfigureAwait(false) so it never
  depends on the ReceiveAsync handler's ActorTaskScheduler (the PostStop
  abandon-and-revert report must run after the actor stopped).
- DriverInstanceActorAckConfirmationTests: a confirmation that faults after the
  handler returned is reported; one pending at actor stop is reported; a
  confirmed ack reports nothing (positive control).
- Deterministic tracker matching test (Raise/Clear/Retrigger/other alarm do not
  confirm; case-insensitive match does).
- AdminUI min/max on the confirm-timeout field; Galaxy.md documents the
  reconnect, UNACK_RTN-as-Clear and RPC-budget limits.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 13:11:49 -05:00

409 lines
18 KiB
C#

using Microsoft.Extensions.Logging;
using Shouldly;
using Xunit;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// lmxopcua#869 — mxaccessgw answers <c>AcknowledgeAlarm</c> OK both when the <c>AckMsg</c> write was APPLIED and
/// when it was only SENT (no <c>OnWriteComplete</c> inside the worker's window; the reply then carries a
/// diagnostic). The H6 live gate saw an unconfirmed ack reported Acknowledged while Galaxy never applied it.
/// The driver now treats an unconfirmed OK as pending and confirms it against Galaxy's own acknowledged
/// transition on the alarm feed, bounded by <c>AlarmAck.ConfirmTimeoutMs</c>.
/// </summary>
/// <remarks>
/// The "fake gateway" is a scripted <see cref="AcknowledgeAlarmReply"/> run through the production
/// <see cref="GatewayGalaxyAlarmAcknowledger.EvaluateReply"/> — the gRPC transport seam is internal to the
/// client package, so the reply mapping is the part under test.
/// </remarks>
public sealed class GalaxyAckConfirmationTests
{
private const string AlarmRef = "Tank01.Level.HiHi";
private const string UnconfirmedDiagnostic =
"AckMsg write to 'Tank01.Level.HiHi.AckMsg' sent; OnWriteComplete not observed within 5000 ms.";
private static readonly GalaxyAckCredential Alice =
new("alice", "CORP", "alice-pw", GalaxyAckCredentialSource.ApiKey);
// ------------------------------------------------------------------ reply mapping
[Fact]
public void EvaluateReply_ok_without_diagnostic_is_applied()
{
var logger = new CapturingLogger();
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(AppliedReply(), AlarmRef, Alice, logger);
outcome.IsApplied.ShouldBeTrue();
logger.Entries.ShouldBeEmpty();
}
/// <summary>The H6 reply: OK + hresult 0 + a "sent; not observed" diagnostic is UNCONFIRMED, and the
/// diagnostic is logged. Before #869 the same reply returned silently as success.</summary>
[Fact]
public void EvaluateReply_ok_with_diagnostic_is_unconfirmed_and_logs_the_diagnostic()
{
var logger = new CapturingLogger();
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(UnconfirmedReply(), AlarmRef, Alice, logger);
outcome.IsApplied.ShouldBeFalse();
outcome.Diagnostic.ShouldBe(UnconfirmedDiagnostic);
logger.AllText.ShouldContain(UnconfirmedDiagnostic);
}
[Fact]
public void EvaluateReply_native_refusal_logs_the_diagnostic_and_refuses()
{
var reply = new AcknowledgeAlarmReply
{
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
Hresult = 1008,
DiagnosticMessage = "AckMsg write to 'Tank01.Level.HiHi.AckMsg' rejected: category SecurityError, detail 1008.",
};
var logger = new CapturingLogger();
Should.Throw<DriverOperationRefusedException>(
() => GatewayGalaxyAlarmAcknowledger.EvaluateReply(reply, AlarmRef, Alice, logger));
logger.AllText.ShouldContain("rejected: category SecurityError");
}
[Fact]
public void EvaluateReply_scrubs_the_password_from_the_diagnostic()
{
var reply = new AcknowledgeAlarmReply
{
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
Hresult = 0,
DiagnosticMessage = "sent as alice/alice-pw; not observed",
};
var logger = new CapturingLogger();
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(reply, AlarmRef, Alice, logger);
outcome.Diagnostic.ShouldNotContain("alice-pw");
logger.AllText.ShouldNotContain("alice-pw");
}
// ------------------------------------------------------------------ driver: confirmation
/// <summary>Applied OK → acknowledged: nothing pending, the confirmation completes at once.</summary>
[Fact]
public async Task Applied_ack_is_confirmed_at_once()
{
var feed = new FakeAlarmFeed();
var gateway = new ScriptedGateway(AppliedReply());
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 30_000);
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
var request = Ack(AlarmRef);
await driver.AcknowledgeAsync(request, CancellationToken.None);
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
confirm.IsCompletedSuccessfully.ShouldBeTrue("an applied ack needs no confirmation");
gateway.Calls.ShouldBe(1);
}
/// <summary>
/// Unconfirmed OK + Galaxy's acknowledged transition on the feed → acknowledged, end to end through the
/// driver's feed hook (non-matching transitions are emitted first; that they do NOT confirm is pinned
/// synchronously by <see cref="Tracker_confirms_only_on_this_alarms_acknowledged_transition"/>).
/// </summary>
[Fact]
public async Task Unconfirmed_ack_is_confirmed_by_the_acknowledged_transition()
{
var feed = new FakeAlarmFeed();
var gateway = new ScriptedGateway(UnconfirmedReply());
var logger = new CapturingLogger();
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 120_000, logger);
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
var request = Ack(AlarmRef);
await driver.AcknowledgeAsync(request, CancellationToken.None);
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
feed.Emit(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Raise));
feed.Emit(Transition(AlarmRef.ToUpperInvariant(), GalaxyAlarmTransitionKind.Acknowledge));
await TestWait.ForAsync(confirm, "the acknowledged transition must confirm the pending ack");
logger.AllText.ShouldContain(UnconfirmedDiagnostic);
}
/// <summary>
/// The feed and the unary reply race: Galaxy's acknowledged transition can arrive BEFORE the RPC answers
/// "unconfirmed". The watch is registered before the RPC, so that early transition still confirms — with a
/// 1 ms window, a lost transition would time out instead.
/// </summary>
[Fact]
public async Task Acknowledged_transition_that_overtakes_the_reply_still_confirms()
{
var feed = new FakeAlarmFeed();
var gateway = new ScriptedGateway(UnconfirmedReply())
{
DuringCall = () => feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Acknowledge)),
};
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 1);
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
var request = Ack(AlarmRef);
await driver.AcknowledgeAsync(request, CancellationToken.None);
await driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
}
/// <summary>
/// Unconfirmed OK + no acknowledged transition → the confirmation fails with a refusal naming the timeout and
/// the gateway diagnostic (the runtime reverts the condition on it), and a Warning carrying the diagnostic is
/// logged. Transitions that must not confirm (another alarm's ack, this alarm's retrigger) are emitted during
/// the wait. Before #869 this ack counted as acknowledged.
/// </summary>
[Fact]
public async Task Unconfirmed_ack_without_confirmation_fails_with_the_diagnostic()
{
var feed = new FakeAlarmFeed();
var gateway = new ScriptedGateway(UnconfirmedReply());
var logger = new CapturingLogger();
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 300, logger);
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
var request = Ack(AlarmRef);
await driver.AcknowledgeAsync(request, CancellationToken.None);
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
feed.Emit(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Retrigger));
var ex = await Should.ThrowAsync<DriverOperationRefusedException>(() => confirm);
ex.Message.ShouldContain(AlarmRef);
ex.Message.ShouldContain("BadTimeout");
ex.Message.ShouldContain(UnconfirmedDiagnostic);
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Text.Contains(UnconfirmedDiagnostic));
}
/// <summary>A refused ack (non-zero native code) throws from AcknowledgeAsync — the existing revert path —
/// and leaves nothing pending to confirm.</summary>
[Fact]
public async Task Refused_ack_throws_and_leaves_nothing_pending()
{
var feed = new FakeAlarmFeed();
var gateway = new ScriptedGateway(new AcknowledgeAlarmReply
{
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
Hresult = unchecked((int)0x80070005),
});
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 30_000);
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
var request = Ack(AlarmRef);
await Should.ThrowAsync<DriverOperationRefusedException>(
() => driver.AcknowledgeAsync(request, CancellationToken.None));
driver.ConfirmAcknowledgedAsync(request, CancellationToken.None).IsCompletedSuccessfully.ShouldBeTrue();
driver.PendingAckConfirmations.ShouldBe(0);
}
/// <summary><c>ConfirmTimeoutMs = 0</c> is the opt-out: an unconfirmed ack counts as acknowledged, with a Warning.</summary>
[Fact]
public async Task Zero_timeout_counts_an_unconfirmed_ack_as_acknowledged_with_a_warning()
{
var feed = new FakeAlarmFeed();
var gateway = new ScriptedGateway(UnconfirmedReply());
var logger = new CapturingLogger();
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 0, logger);
var request = Ack(AlarmRef);
await driver.AcknowledgeAsync(request, CancellationToken.None);
driver.ConfirmAcknowledgedAsync(request, CancellationToken.None).IsCompletedSuccessfully.ShouldBeTrue();
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Text.Contains("confirmation is disabled"));
driver.PendingAckConfirmations.ShouldBe(0);
}
/// <summary>Each confirmation is for ITS request instance: confirming one leaves no watch behind, and a later
/// ack of the same alarm is confirmed independently.</summary>
[Fact]
public async Task Confirmations_release_their_watches()
{
var feed = new FakeAlarmFeed();
var gateway = new ScriptedGateway(UnconfirmedReply());
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 120_000);
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
for (var i = 0; i < 2; i++)
{
var request = Ack(AlarmRef);
await driver.AcknowledgeAsync(request, CancellationToken.None);
driver.PendingAckConfirmations.ShouldBe(1);
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.ActiveAcknowledged));
await TestWait.ForAsync(confirm, $"ack #{i} must be confirmed");
driver.PendingAckConfirmations.ShouldBe(0);
}
}
/// <summary>
/// Tracker matching rules, checked synchronously: only an Acknowledge / ActiveAcknowledged transition of the
/// SAME alarm (case-insensitive) confirms. Positive control: the matching transition does flip it.
/// </summary>
[Fact]
public void Tracker_confirms_only_on_this_alarms_acknowledged_transition()
{
var tracker = new GalaxyAckConfirmationTracker(TimeSpan.FromSeconds(30), new CapturingLogger(), "drv");
var watch = tracker.Begin(Ack(AlarmRef)[0], AlarmRef);
tracker.Arm(watch, UnconfirmedDiagnostic);
foreach (var kind in new[]
{
GalaxyAlarmTransitionKind.Raise, GalaxyAlarmTransitionKind.Clear,
GalaxyAlarmTransitionKind.Retrigger, GalaxyAlarmTransitionKind.Unspecified,
})
{
tracker.Observe(Transition(AlarmRef, kind));
}
tracker.Observe(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
tracker.Observe(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.ActiveAcknowledged));
watch.IsConfirmed.ShouldBeFalse("no transition so far says THIS alarm is acknowledged");
tracker.Observe(Transition(AlarmRef.ToLowerInvariant(), GalaxyAlarmTransitionKind.Acknowledge));
watch.IsConfirmed.ShouldBeTrue();
}
[Fact]
public void Options_default_to_ten_seconds_and_map_negative_to_the_default()
{
new GalaxyAlarmAckOptions().ConfirmTimeout.ShouldBe(TimeSpan.FromSeconds(10));
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = -5 }.ConfirmTimeout.ShouldBe(TimeSpan.FromSeconds(10));
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 0 }.ConfirmTimeout.ShouldBe(TimeSpan.Zero);
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 999_999 }.ConfirmTimeout.ShouldBe(TimeSpan.FromMinutes(2));
}
[Fact]
public void Factory_binds_ConfirmTimeoutMs_and_defaults_when_absent()
{
const string Json = """
{ "gateway": { "endpoint": "https://gw:5001", "apiKeySecretRef": "dev:k" },
"mxAccess": { "clientName": "C" }, "alarmAck": { "confirmTimeoutMs": 2500 } }
""";
using var withKnob = GalaxyDriverFactoryExtensions.CreateInstance("g", Json);
withKnob.AckConfirmationWindow.ShouldBe(TimeSpan.FromMilliseconds(2500));
using var without = GalaxyDriverFactoryExtensions.CreateInstance(
"g", """{ "gateway": { "endpoint": "https://gw:5001", "apiKeySecretRef": "dev:k" }, "mxAccess": { "clientName": "C" } }""");
without.AckConfirmationWindow.ShouldBe(TimeSpan.FromSeconds(10));
}
// ------------------------------------------------------------------ helpers
private static AcknowledgeAlarmReply AppliedReply() => new()
{
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
Hresult = 0,
};
private static AcknowledgeAlarmReply UnconfirmedReply() => new()
{
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
Hresult = 0,
DiagnosticMessage = UnconfirmedDiagnostic,
};
private static AlarmAcknowledgeRequest[] Ack(string conditionId) =>
[new AlarmAcknowledgeRequest(SourceNodeId: "Tank01", ConditionId: conditionId, Comment: "c")];
private static GalaxyDriver NewDriver(
IGalaxyAlarmFeed feed, IGalaxyAlarmAcknowledger acknowledger, int confirmTimeoutMs, CapturingLogger? logger = null)
{
var options = new GalaxyDriverOptions(
new GalaxyGatewayOptions("http://localhost:5000", "literal-api-key"),
new GalaxyMxAccessOptions("AckConfirmTest"),
new GalaxyRepositoryOptions(),
new GalaxyReconnectOptions())
{
AlarmAck = new GalaxyAlarmAckOptions { ConfirmTimeoutMs = confirmTimeoutMs },
};
return new GalaxyDriver(
driverInstanceId: "drv-ack",
options: options,
hierarchySource: null,
alarmAcknowledger: acknowledger,
alarmFeed: feed,
logger: logger is null ? null : new TypedLogger<GalaxyDriver>(logger),
ackCredentialResolver: FixedAckCredentialResolver.Fallback());
}
private static GalaxyAlarmTransition Transition(string alarmRef, GalaxyAlarmTransitionKind kind) => new(
AlarmFullReference: alarmRef,
SourceObjectReference: "Tank01",
AlarmTypeName: "AnalogLimitAlarm.HiHi",
TransitionKind: kind,
SeverityBucket: AlarmSeverity.High,
OpcUaSeverity: 700,
RawMxAccessSeverity: 500,
OriginalRaiseTimestampUtc: null,
TransitionTimestampUtc: DateTime.UtcNow,
OperatorUser: string.Empty,
OperatorComment: string.Empty,
Category: "Process",
Description: "Tank 01 high-high level");
/// <summary>
/// A fake gateway: every call answers with <paramref name="reply"/>, mapped by the production
/// <see cref="GatewayGalaxyAlarmAcknowledger.EvaluateReply"/>. <see cref="DuringCall"/> runs while the RPC is
/// "in flight" (before the reply is evaluated).
/// </summary>
private sealed class ScriptedGateway(AcknowledgeAlarmReply reply) : IGalaxyAlarmAcknowledger
{
private int _calls;
public Action? DuringCall { get; init; }
public int Calls => Volatile.Read(ref _calls);
public Task<GalaxyAckOutcome> AcknowledgeAsync(
string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
{
Interlocked.Increment(ref _calls);
DuringCall?.Invoke();
try
{
return Task.FromResult(GatewayGalaxyAlarmAcknowledger.EvaluateReply(
reply, alarmFullReference, credential, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance));
}
catch (Exception ex)
{
return Task.FromException<GalaxyAckOutcome>(ex);
}
}
}
private sealed class FakeAlarmFeed : IGalaxyAlarmFeed
{
public event EventHandler<GalaxyAlarmTransition>? OnAlarmTransition;
public void Start() { }
public void Emit(GalaxyAlarmTransition transition) => OnAlarmTransition?.Invoke(this, transition);
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
private sealed class TypedLogger<T>(CapturingLogger inner) : ILogger<T>
{
private readonly Lock _gate = new();
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel l, EventId id, TState s, Exception? e, Func<TState, Exception?, string> f)
{
lock (_gate) inner.Log(l, id, s, e, f);
}
}
}