77872b1d1c
- Start the ack confirmation with Task.Run + ConfigureAwait(false) so it never depends on the ReceiveAsync handler's ActorTaskScheduler (the PostStop abandon-and-revert report must run after the actor stopped). - DriverInstanceActorAckConfirmationTests: a confirmation that faults after the handler returned is reported; one pending at actor stop is reported; a confirmed ack reports nothing (positive control). - Deterministic tracker matching test (Raise/Clear/Retrigger/other alarm do not confirm; case-insensitive match does). - AdminUI min/max on the confirm-timeout field; Galaxy.md documents the reconnect, UNACK_RTN-as-Clear and RPC-budget limits. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
409 lines
18 KiB
C#
409 lines
18 KiB
C#
using Microsoft.Extensions.Logging;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
|
|
|
/// <summary>
|
|
/// lmxopcua#869 — mxaccessgw answers <c>AcknowledgeAlarm</c> OK both when the <c>AckMsg</c> write was APPLIED and
|
|
/// when it was only SENT (no <c>OnWriteComplete</c> inside the worker's window; the reply then carries a
|
|
/// diagnostic). The H6 live gate saw an unconfirmed ack reported Acknowledged while Galaxy never applied it.
|
|
/// The driver now treats an unconfirmed OK as pending and confirms it against Galaxy's own acknowledged
|
|
/// transition on the alarm feed, bounded by <c>AlarmAck.ConfirmTimeoutMs</c>.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// The "fake gateway" is a scripted <see cref="AcknowledgeAlarmReply"/> run through the production
|
|
/// <see cref="GatewayGalaxyAlarmAcknowledger.EvaluateReply"/> — the gRPC transport seam is internal to the
|
|
/// client package, so the reply mapping is the part under test.
|
|
/// </remarks>
|
|
public sealed class GalaxyAckConfirmationTests
|
|
{
|
|
private const string AlarmRef = "Tank01.Level.HiHi";
|
|
|
|
private const string UnconfirmedDiagnostic =
|
|
"AckMsg write to 'Tank01.Level.HiHi.AckMsg' sent; OnWriteComplete not observed within 5000 ms.";
|
|
|
|
private static readonly GalaxyAckCredential Alice =
|
|
new("alice", "CORP", "alice-pw", GalaxyAckCredentialSource.ApiKey);
|
|
|
|
// ------------------------------------------------------------------ reply mapping
|
|
|
|
[Fact]
|
|
public void EvaluateReply_ok_without_diagnostic_is_applied()
|
|
{
|
|
var logger = new CapturingLogger();
|
|
|
|
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(AppliedReply(), AlarmRef, Alice, logger);
|
|
|
|
outcome.IsApplied.ShouldBeTrue();
|
|
logger.Entries.ShouldBeEmpty();
|
|
}
|
|
|
|
/// <summary>The H6 reply: OK + hresult 0 + a "sent; not observed" diagnostic is UNCONFIRMED, and the
|
|
/// diagnostic is logged. Before #869 the same reply returned silently as success.</summary>
|
|
[Fact]
|
|
public void EvaluateReply_ok_with_diagnostic_is_unconfirmed_and_logs_the_diagnostic()
|
|
{
|
|
var logger = new CapturingLogger();
|
|
|
|
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(UnconfirmedReply(), AlarmRef, Alice, logger);
|
|
|
|
outcome.IsApplied.ShouldBeFalse();
|
|
outcome.Diagnostic.ShouldBe(UnconfirmedDiagnostic);
|
|
logger.AllText.ShouldContain(UnconfirmedDiagnostic);
|
|
}
|
|
|
|
[Fact]
|
|
public void EvaluateReply_native_refusal_logs_the_diagnostic_and_refuses()
|
|
{
|
|
var reply = new AcknowledgeAlarmReply
|
|
{
|
|
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
|
Hresult = 1008,
|
|
DiagnosticMessage = "AckMsg write to 'Tank01.Level.HiHi.AckMsg' rejected: category SecurityError, detail 1008.",
|
|
};
|
|
var logger = new CapturingLogger();
|
|
|
|
Should.Throw<DriverOperationRefusedException>(
|
|
() => GatewayGalaxyAlarmAcknowledger.EvaluateReply(reply, AlarmRef, Alice, logger));
|
|
|
|
logger.AllText.ShouldContain("rejected: category SecurityError");
|
|
}
|
|
|
|
[Fact]
|
|
public void EvaluateReply_scrubs_the_password_from_the_diagnostic()
|
|
{
|
|
var reply = new AcknowledgeAlarmReply
|
|
{
|
|
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
|
Hresult = 0,
|
|
DiagnosticMessage = "sent as alice/alice-pw; not observed",
|
|
};
|
|
var logger = new CapturingLogger();
|
|
|
|
var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(reply, AlarmRef, Alice, logger);
|
|
|
|
outcome.Diagnostic.ShouldNotContain("alice-pw");
|
|
logger.AllText.ShouldNotContain("alice-pw");
|
|
}
|
|
|
|
// ------------------------------------------------------------------ driver: confirmation
|
|
|
|
/// <summary>Applied OK → acknowledged: nothing pending, the confirmation completes at once.</summary>
|
|
[Fact]
|
|
public async Task Applied_ack_is_confirmed_at_once()
|
|
{
|
|
var feed = new FakeAlarmFeed();
|
|
var gateway = new ScriptedGateway(AppliedReply());
|
|
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 30_000);
|
|
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
|
var request = Ack(AlarmRef);
|
|
|
|
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
|
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
|
|
|
confirm.IsCompletedSuccessfully.ShouldBeTrue("an applied ack needs no confirmation");
|
|
gateway.Calls.ShouldBe(1);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Unconfirmed OK + Galaxy's acknowledged transition on the feed → acknowledged, end to end through the
|
|
/// driver's feed hook (non-matching transitions are emitted first; that they do NOT confirm is pinned
|
|
/// synchronously by <see cref="Tracker_confirms_only_on_this_alarms_acknowledged_transition"/>).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Unconfirmed_ack_is_confirmed_by_the_acknowledged_transition()
|
|
{
|
|
var feed = new FakeAlarmFeed();
|
|
var gateway = new ScriptedGateway(UnconfirmedReply());
|
|
var logger = new CapturingLogger();
|
|
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 120_000, logger);
|
|
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
|
var request = Ack(AlarmRef);
|
|
|
|
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
|
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
|
|
|
feed.Emit(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
|
|
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Raise));
|
|
feed.Emit(Transition(AlarmRef.ToUpperInvariant(), GalaxyAlarmTransitionKind.Acknowledge));
|
|
|
|
await TestWait.ForAsync(confirm, "the acknowledged transition must confirm the pending ack");
|
|
logger.AllText.ShouldContain(UnconfirmedDiagnostic);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The feed and the unary reply race: Galaxy's acknowledged transition can arrive BEFORE the RPC answers
|
|
/// "unconfirmed". The watch is registered before the RPC, so that early transition still confirms — with a
|
|
/// 1 ms window, a lost transition would time out instead.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Acknowledged_transition_that_overtakes_the_reply_still_confirms()
|
|
{
|
|
var feed = new FakeAlarmFeed();
|
|
var gateway = new ScriptedGateway(UnconfirmedReply())
|
|
{
|
|
DuringCall = () => feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Acknowledge)),
|
|
};
|
|
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 1);
|
|
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
|
var request = Ack(AlarmRef);
|
|
|
|
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
|
await driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Unconfirmed OK + no acknowledged transition → the confirmation fails with a refusal naming the timeout and
|
|
/// the gateway diagnostic (the runtime reverts the condition on it), and a Warning carrying the diagnostic is
|
|
/// logged. Transitions that must not confirm (another alarm's ack, this alarm's retrigger) are emitted during
|
|
/// the wait. Before #869 this ack counted as acknowledged.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Unconfirmed_ack_without_confirmation_fails_with_the_diagnostic()
|
|
{
|
|
var feed = new FakeAlarmFeed();
|
|
var gateway = new ScriptedGateway(UnconfirmedReply());
|
|
var logger = new CapturingLogger();
|
|
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 300, logger);
|
|
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
|
var request = Ack(AlarmRef);
|
|
|
|
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
|
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
|
feed.Emit(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
|
|
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Retrigger));
|
|
|
|
var ex = await Should.ThrowAsync<DriverOperationRefusedException>(() => confirm);
|
|
ex.Message.ShouldContain(AlarmRef);
|
|
ex.Message.ShouldContain("BadTimeout");
|
|
ex.Message.ShouldContain(UnconfirmedDiagnostic);
|
|
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Text.Contains(UnconfirmedDiagnostic));
|
|
}
|
|
|
|
/// <summary>A refused ack (non-zero native code) throws from AcknowledgeAsync — the existing revert path —
|
|
/// and leaves nothing pending to confirm.</summary>
|
|
[Fact]
|
|
public async Task Refused_ack_throws_and_leaves_nothing_pending()
|
|
{
|
|
var feed = new FakeAlarmFeed();
|
|
var gateway = new ScriptedGateway(new AcknowledgeAlarmReply
|
|
{
|
|
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
|
Hresult = unchecked((int)0x80070005),
|
|
});
|
|
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 30_000);
|
|
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
|
var request = Ack(AlarmRef);
|
|
|
|
await Should.ThrowAsync<DriverOperationRefusedException>(
|
|
() => driver.AcknowledgeAsync(request, CancellationToken.None));
|
|
|
|
driver.ConfirmAcknowledgedAsync(request, CancellationToken.None).IsCompletedSuccessfully.ShouldBeTrue();
|
|
driver.PendingAckConfirmations.ShouldBe(0);
|
|
}
|
|
|
|
/// <summary><c>ConfirmTimeoutMs = 0</c> is the opt-out: an unconfirmed ack counts as acknowledged, with a Warning.</summary>
|
|
[Fact]
|
|
public async Task Zero_timeout_counts_an_unconfirmed_ack_as_acknowledged_with_a_warning()
|
|
{
|
|
var feed = new FakeAlarmFeed();
|
|
var gateway = new ScriptedGateway(UnconfirmedReply());
|
|
var logger = new CapturingLogger();
|
|
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 0, logger);
|
|
var request = Ack(AlarmRef);
|
|
|
|
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
|
|
|
driver.ConfirmAcknowledgedAsync(request, CancellationToken.None).IsCompletedSuccessfully.ShouldBeTrue();
|
|
logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Text.Contains("confirmation is disabled"));
|
|
driver.PendingAckConfirmations.ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>Each confirmation is for ITS request instance: confirming one leaves no watch behind, and a later
|
|
/// ack of the same alarm is confirmed independently.</summary>
|
|
[Fact]
|
|
public async Task Confirmations_release_their_watches()
|
|
{
|
|
var feed = new FakeAlarmFeed();
|
|
var gateway = new ScriptedGateway(UnconfirmedReply());
|
|
using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 120_000);
|
|
await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None);
|
|
|
|
for (var i = 0; i < 2; i++)
|
|
{
|
|
var request = Ack(AlarmRef);
|
|
await driver.AcknowledgeAsync(request, CancellationToken.None);
|
|
driver.PendingAckConfirmations.ShouldBe(1);
|
|
var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None);
|
|
feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.ActiveAcknowledged));
|
|
await TestWait.ForAsync(confirm, $"ack #{i} must be confirmed");
|
|
driver.PendingAckConfirmations.ShouldBe(0);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tracker matching rules, checked synchronously: only an Acknowledge / ActiveAcknowledged transition of the
|
|
/// SAME alarm (case-insensitive) confirms. Positive control: the matching transition does flip it.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Tracker_confirms_only_on_this_alarms_acknowledged_transition()
|
|
{
|
|
var tracker = new GalaxyAckConfirmationTracker(TimeSpan.FromSeconds(30), new CapturingLogger(), "drv");
|
|
var watch = tracker.Begin(Ack(AlarmRef)[0], AlarmRef);
|
|
tracker.Arm(watch, UnconfirmedDiagnostic);
|
|
|
|
foreach (var kind in new[]
|
|
{
|
|
GalaxyAlarmTransitionKind.Raise, GalaxyAlarmTransitionKind.Clear,
|
|
GalaxyAlarmTransitionKind.Retrigger, GalaxyAlarmTransitionKind.Unspecified,
|
|
})
|
|
{
|
|
tracker.Observe(Transition(AlarmRef, kind));
|
|
}
|
|
tracker.Observe(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge));
|
|
tracker.Observe(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.ActiveAcknowledged));
|
|
watch.IsConfirmed.ShouldBeFalse("no transition so far says THIS alarm is acknowledged");
|
|
|
|
tracker.Observe(Transition(AlarmRef.ToLowerInvariant(), GalaxyAlarmTransitionKind.Acknowledge));
|
|
watch.IsConfirmed.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Options_default_to_ten_seconds_and_map_negative_to_the_default()
|
|
{
|
|
new GalaxyAlarmAckOptions().ConfirmTimeout.ShouldBe(TimeSpan.FromSeconds(10));
|
|
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = -5 }.ConfirmTimeout.ShouldBe(TimeSpan.FromSeconds(10));
|
|
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 0 }.ConfirmTimeout.ShouldBe(TimeSpan.Zero);
|
|
new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 999_999 }.ConfirmTimeout.ShouldBe(TimeSpan.FromMinutes(2));
|
|
}
|
|
|
|
[Fact]
|
|
public void Factory_binds_ConfirmTimeoutMs_and_defaults_when_absent()
|
|
{
|
|
const string Json = """
|
|
{ "gateway": { "endpoint": "https://gw:5001", "apiKeySecretRef": "dev:k" },
|
|
"mxAccess": { "clientName": "C" }, "alarmAck": { "confirmTimeoutMs": 2500 } }
|
|
""";
|
|
using var withKnob = GalaxyDriverFactoryExtensions.CreateInstance("g", Json);
|
|
withKnob.AckConfirmationWindow.ShouldBe(TimeSpan.FromMilliseconds(2500));
|
|
|
|
using var without = GalaxyDriverFactoryExtensions.CreateInstance(
|
|
"g", """{ "gateway": { "endpoint": "https://gw:5001", "apiKeySecretRef": "dev:k" }, "mxAccess": { "clientName": "C" } }""");
|
|
without.AckConfirmationWindow.ShouldBe(TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
// ------------------------------------------------------------------ helpers
|
|
|
|
private static AcknowledgeAlarmReply AppliedReply() => new()
|
|
{
|
|
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
|
Hresult = 0,
|
|
};
|
|
|
|
private static AcknowledgeAlarmReply UnconfirmedReply() => new()
|
|
{
|
|
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
|
Hresult = 0,
|
|
DiagnosticMessage = UnconfirmedDiagnostic,
|
|
};
|
|
|
|
private static AlarmAcknowledgeRequest[] Ack(string conditionId) =>
|
|
[new AlarmAcknowledgeRequest(SourceNodeId: "Tank01", ConditionId: conditionId, Comment: "c")];
|
|
|
|
private static GalaxyDriver NewDriver(
|
|
IGalaxyAlarmFeed feed, IGalaxyAlarmAcknowledger acknowledger, int confirmTimeoutMs, CapturingLogger? logger = null)
|
|
{
|
|
var options = new GalaxyDriverOptions(
|
|
new GalaxyGatewayOptions("http://localhost:5000", "literal-api-key"),
|
|
new GalaxyMxAccessOptions("AckConfirmTest"),
|
|
new GalaxyRepositoryOptions(),
|
|
new GalaxyReconnectOptions())
|
|
{
|
|
AlarmAck = new GalaxyAlarmAckOptions { ConfirmTimeoutMs = confirmTimeoutMs },
|
|
};
|
|
return new GalaxyDriver(
|
|
driverInstanceId: "drv-ack",
|
|
options: options,
|
|
hierarchySource: null,
|
|
alarmAcknowledger: acknowledger,
|
|
alarmFeed: feed,
|
|
logger: logger is null ? null : new TypedLogger<GalaxyDriver>(logger),
|
|
ackCredentialResolver: FixedAckCredentialResolver.Fallback());
|
|
}
|
|
|
|
private static GalaxyAlarmTransition Transition(string alarmRef, GalaxyAlarmTransitionKind kind) => new(
|
|
AlarmFullReference: alarmRef,
|
|
SourceObjectReference: "Tank01",
|
|
AlarmTypeName: "AnalogLimitAlarm.HiHi",
|
|
TransitionKind: kind,
|
|
SeverityBucket: AlarmSeverity.High,
|
|
OpcUaSeverity: 700,
|
|
RawMxAccessSeverity: 500,
|
|
OriginalRaiseTimestampUtc: null,
|
|
TransitionTimestampUtc: DateTime.UtcNow,
|
|
OperatorUser: string.Empty,
|
|
OperatorComment: string.Empty,
|
|
Category: "Process",
|
|
Description: "Tank 01 high-high level");
|
|
|
|
/// <summary>
|
|
/// A fake gateway: every call answers with <paramref name="reply"/>, mapped by the production
|
|
/// <see cref="GatewayGalaxyAlarmAcknowledger.EvaluateReply"/>. <see cref="DuringCall"/> runs while the RPC is
|
|
/// "in flight" (before the reply is evaluated).
|
|
/// </summary>
|
|
private sealed class ScriptedGateway(AcknowledgeAlarmReply reply) : IGalaxyAlarmAcknowledger
|
|
{
|
|
private int _calls;
|
|
|
|
public Action? DuringCall { get; init; }
|
|
|
|
public int Calls => Volatile.Read(ref _calls);
|
|
|
|
public Task<GalaxyAckOutcome> AcknowledgeAsync(
|
|
string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref _calls);
|
|
DuringCall?.Invoke();
|
|
try
|
|
{
|
|
return Task.FromResult(GatewayGalaxyAlarmAcknowledger.EvaluateReply(
|
|
reply, alarmFullReference, credential, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
return Task.FromException<GalaxyAckOutcome>(ex);
|
|
}
|
|
}
|
|
}
|
|
|
|
private sealed class FakeAlarmFeed : IGalaxyAlarmFeed
|
|
{
|
|
public event EventHandler<GalaxyAlarmTransition>? OnAlarmTransition;
|
|
|
|
public void Start() { }
|
|
|
|
public void Emit(GalaxyAlarmTransition transition) => OnAlarmTransition?.Invoke(this, transition);
|
|
|
|
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
|
}
|
|
|
|
private sealed class TypedLogger<T>(CapturingLogger inner) : ILogger<T>
|
|
{
|
|
private readonly Lock _gate = new();
|
|
|
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
|
|
|
public bool IsEnabled(LogLevel logLevel) => true;
|
|
|
|
public void Log<TState>(LogLevel l, EventId id, TState s, Exception? e, Func<TState, Exception?, string> f)
|
|
{
|
|
lock (_gate) inner.Log(l, id, s, e, f);
|
|
}
|
|
}
|
|
}
|