using Microsoft.Extensions.Logging; using Shouldly; using Xunit; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// /// lmxopcua#869 — mxaccessgw answers AcknowledgeAlarm OK both when the AckMsg write was APPLIED and /// when it was only SENT (no OnWriteComplete inside the worker's window; the reply then carries a /// diagnostic). The H6 live gate saw an unconfirmed ack reported Acknowledged while Galaxy never applied it. /// The driver now treats an unconfirmed OK as pending and confirms it against Galaxy's own acknowledged /// transition on the alarm feed, bounded by AlarmAck.ConfirmTimeoutMs. /// /// /// The "fake gateway" is a scripted run through the production /// — the gRPC transport seam is internal to the /// client package, so the reply mapping is the part under test. /// public sealed class GalaxyAckConfirmationTests { private const string AlarmRef = "Tank01.Level.HiHi"; private const string UnconfirmedDiagnostic = "AckMsg write to 'Tank01.Level.HiHi.AckMsg' sent; OnWriteComplete not observed within 5000 ms."; private static readonly GalaxyAckCredential Alice = new("alice", "CORP", "alice-pw", GalaxyAckCredentialSource.ApiKey); // ------------------------------------------------------------------ reply mapping [Fact] public void EvaluateReply_ok_without_diagnostic_is_applied() { var logger = new CapturingLogger(); var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(AppliedReply(), AlarmRef, Alice, logger); outcome.IsApplied.ShouldBeTrue(); logger.Entries.ShouldBeEmpty(); } /// The H6 reply: OK + hresult 0 + a "sent; not observed" diagnostic is UNCONFIRMED, and the /// diagnostic is logged. Before #869 the same reply returned silently as success. [Fact] public void EvaluateReply_ok_with_diagnostic_is_unconfirmed_and_logs_the_diagnostic() { var logger = new CapturingLogger(); var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(UnconfirmedReply(), AlarmRef, Alice, logger); outcome.IsApplied.ShouldBeFalse(); outcome.Diagnostic.ShouldBe(UnconfirmedDiagnostic); logger.AllText.ShouldContain(UnconfirmedDiagnostic); } [Fact] public void EvaluateReply_native_refusal_logs_the_diagnostic_and_refuses() { var reply = new AcknowledgeAlarmReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok }, Hresult = 1008, DiagnosticMessage = "AckMsg write to 'Tank01.Level.HiHi.AckMsg' rejected: category SecurityError, detail 1008.", }; var logger = new CapturingLogger(); Should.Throw( () => GatewayGalaxyAlarmAcknowledger.EvaluateReply(reply, AlarmRef, Alice, logger)); logger.AllText.ShouldContain("rejected: category SecurityError"); } [Fact] public void EvaluateReply_scrubs_the_password_from_the_diagnostic() { var reply = new AcknowledgeAlarmReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok }, Hresult = 0, DiagnosticMessage = "sent as alice/alice-pw; not observed", }; var logger = new CapturingLogger(); var outcome = GatewayGalaxyAlarmAcknowledger.EvaluateReply(reply, AlarmRef, Alice, logger); outcome.Diagnostic.ShouldNotContain("alice-pw"); logger.AllText.ShouldNotContain("alice-pw"); } // ------------------------------------------------------------------ driver: confirmation /// Applied OK → acknowledged: nothing pending, the confirmation completes at once. [Fact] public async Task Applied_ack_is_confirmed_at_once() { var feed = new FakeAlarmFeed(); var gateway = new ScriptedGateway(AppliedReply()); using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 30_000); await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None); var request = Ack(AlarmRef); await driver.AcknowledgeAsync(request, CancellationToken.None); var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None); confirm.IsCompletedSuccessfully.ShouldBeTrue("an applied ack needs no confirmation"); gateway.Calls.ShouldBe(1); } /// /// Unconfirmed OK + Galaxy's acknowledged transition on the feed → acknowledged, end to end through the /// driver's feed hook (non-matching transitions are emitted first; that they do NOT confirm is pinned /// synchronously by ). /// [Fact] public async Task Unconfirmed_ack_is_confirmed_by_the_acknowledged_transition() { var feed = new FakeAlarmFeed(); var gateway = new ScriptedGateway(UnconfirmedReply()); var logger = new CapturingLogger(); using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 120_000, logger); await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None); var request = Ack(AlarmRef); await driver.AcknowledgeAsync(request, CancellationToken.None); var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None); feed.Emit(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge)); feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Raise)); feed.Emit(Transition(AlarmRef.ToUpperInvariant(), GalaxyAlarmTransitionKind.Acknowledge)); await TestWait.ForAsync(confirm, "the acknowledged transition must confirm the pending ack"); logger.AllText.ShouldContain(UnconfirmedDiagnostic); } /// /// The feed and the unary reply race: Galaxy's acknowledged transition can arrive BEFORE the RPC answers /// "unconfirmed". The watch is registered before the RPC, so that early transition still confirms — with a /// 1 ms window, a lost transition would time out instead. /// [Fact] public async Task Acknowledged_transition_that_overtakes_the_reply_still_confirms() { var feed = new FakeAlarmFeed(); var gateway = new ScriptedGateway(UnconfirmedReply()) { DuringCall = () => feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Acknowledge)), }; using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 1); await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None); var request = Ack(AlarmRef); await driver.AcknowledgeAsync(request, CancellationToken.None); await driver.ConfirmAcknowledgedAsync(request, CancellationToken.None); } /// /// Unconfirmed OK + no acknowledged transition → the confirmation fails with a refusal naming the timeout and /// the gateway diagnostic (the runtime reverts the condition on it), and a Warning carrying the diagnostic is /// logged. Transitions that must not confirm (another alarm's ack, this alarm's retrigger) are emitted during /// the wait. Before #869 this ack counted as acknowledged. /// [Fact] public async Task Unconfirmed_ack_without_confirmation_fails_with_the_diagnostic() { var feed = new FakeAlarmFeed(); var gateway = new ScriptedGateway(UnconfirmedReply()); var logger = new CapturingLogger(); using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 300, logger); await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None); var request = Ack(AlarmRef); await driver.AcknowledgeAsync(request, CancellationToken.None); var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None); feed.Emit(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge)); feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.Retrigger)); var ex = await Should.ThrowAsync(() => confirm); ex.Message.ShouldContain(AlarmRef); ex.Message.ShouldContain("BadTimeout"); ex.Message.ShouldContain(UnconfirmedDiagnostic); logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Text.Contains(UnconfirmedDiagnostic)); } /// A refused ack (non-zero native code) throws from AcknowledgeAsync — the existing revert path — /// and leaves nothing pending to confirm. [Fact] public async Task Refused_ack_throws_and_leaves_nothing_pending() { var feed = new FakeAlarmFeed(); var gateway = new ScriptedGateway(new AcknowledgeAlarmReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok }, Hresult = unchecked((int)0x80070005), }); using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 30_000); await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None); var request = Ack(AlarmRef); await Should.ThrowAsync( () => driver.AcknowledgeAsync(request, CancellationToken.None)); driver.ConfirmAcknowledgedAsync(request, CancellationToken.None).IsCompletedSuccessfully.ShouldBeTrue(); driver.PendingAckConfirmations.ShouldBe(0); } /// ConfirmTimeoutMs = 0 is the opt-out: an unconfirmed ack counts as acknowledged, with a Warning. [Fact] public async Task Zero_timeout_counts_an_unconfirmed_ack_as_acknowledged_with_a_warning() { var feed = new FakeAlarmFeed(); var gateway = new ScriptedGateway(UnconfirmedReply()); var logger = new CapturingLogger(); using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 0, logger); var request = Ack(AlarmRef); await driver.AcknowledgeAsync(request, CancellationToken.None); driver.ConfirmAcknowledgedAsync(request, CancellationToken.None).IsCompletedSuccessfully.ShouldBeTrue(); logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Text.Contains("confirmation is disabled")); driver.PendingAckConfirmations.ShouldBe(0); } /// Each confirmation is for ITS request instance: confirming one leaves no watch behind, and a later /// ack of the same alarm is confirmed independently. [Fact] public async Task Confirmations_release_their_watches() { var feed = new FakeAlarmFeed(); var gateway = new ScriptedGateway(UnconfirmedReply()); using var driver = NewDriver(feed, gateway, confirmTimeoutMs: 120_000); await driver.SubscribeAlarmsAsync([AlarmRef], CancellationToken.None); for (var i = 0; i < 2; i++) { var request = Ack(AlarmRef); await driver.AcknowledgeAsync(request, CancellationToken.None); driver.PendingAckConfirmations.ShouldBe(1); var confirm = driver.ConfirmAcknowledgedAsync(request, CancellationToken.None); feed.Emit(Transition(AlarmRef, GalaxyAlarmTransitionKind.ActiveAcknowledged)); await TestWait.ForAsync(confirm, $"ack #{i} must be confirmed"); driver.PendingAckConfirmations.ShouldBe(0); } } /// /// Tracker matching rules, checked synchronously: only an Acknowledge / ActiveAcknowledged transition of the /// SAME alarm (case-insensitive) confirms. Positive control: the matching transition does flip it. /// [Fact] public void Tracker_confirms_only_on_this_alarms_acknowledged_transition() { var tracker = new GalaxyAckConfirmationTracker(TimeSpan.FromSeconds(30), new CapturingLogger(), "drv"); var watch = tracker.Begin(Ack(AlarmRef)[0], AlarmRef); tracker.Arm(watch, UnconfirmedDiagnostic); foreach (var kind in new[] { GalaxyAlarmTransitionKind.Raise, GalaxyAlarmTransitionKind.Clear, GalaxyAlarmTransitionKind.Retrigger, GalaxyAlarmTransitionKind.Unspecified, }) { tracker.Observe(Transition(AlarmRef, kind)); } tracker.Observe(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.Acknowledge)); tracker.Observe(Transition("Other01.Level.Hi", GalaxyAlarmTransitionKind.ActiveAcknowledged)); watch.IsConfirmed.ShouldBeFalse("no transition so far says THIS alarm is acknowledged"); tracker.Observe(Transition(AlarmRef.ToLowerInvariant(), GalaxyAlarmTransitionKind.Acknowledge)); watch.IsConfirmed.ShouldBeTrue(); } [Fact] public void Options_default_to_ten_seconds_and_map_negative_to_the_default() { new GalaxyAlarmAckOptions().ConfirmTimeout.ShouldBe(TimeSpan.FromSeconds(10)); new GalaxyAlarmAckOptions { ConfirmTimeoutMs = -5 }.ConfirmTimeout.ShouldBe(TimeSpan.FromSeconds(10)); new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 0 }.ConfirmTimeout.ShouldBe(TimeSpan.Zero); new GalaxyAlarmAckOptions { ConfirmTimeoutMs = 999_999 }.ConfirmTimeout.ShouldBe(TimeSpan.FromMinutes(2)); } [Fact] public void Factory_binds_ConfirmTimeoutMs_and_defaults_when_absent() { const string Json = """ { "gateway": { "endpoint": "https://gw:5001", "apiKeySecretRef": "dev:k" }, "mxAccess": { "clientName": "C" }, "alarmAck": { "confirmTimeoutMs": 2500 } } """; using var withKnob = GalaxyDriverFactoryExtensions.CreateInstance("g", Json); withKnob.AckConfirmationWindow.ShouldBe(TimeSpan.FromMilliseconds(2500)); using var without = GalaxyDriverFactoryExtensions.CreateInstance( "g", """{ "gateway": { "endpoint": "https://gw:5001", "apiKeySecretRef": "dev:k" }, "mxAccess": { "clientName": "C" } }"""); without.AckConfirmationWindow.ShouldBe(TimeSpan.FromSeconds(10)); } // ------------------------------------------------------------------ helpers private static AcknowledgeAlarmReply AppliedReply() => new() { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok }, Hresult = 0, }; private static AcknowledgeAlarmReply UnconfirmedReply() => new() { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok }, Hresult = 0, DiagnosticMessage = UnconfirmedDiagnostic, }; private static AlarmAcknowledgeRequest[] Ack(string conditionId) => [new AlarmAcknowledgeRequest(SourceNodeId: "Tank01", ConditionId: conditionId, Comment: "c")]; private static GalaxyDriver NewDriver( IGalaxyAlarmFeed feed, IGalaxyAlarmAcknowledger acknowledger, int confirmTimeoutMs, CapturingLogger? logger = null) { var options = new GalaxyDriverOptions( new GalaxyGatewayOptions("http://localhost:5000", "literal-api-key"), new GalaxyMxAccessOptions("AckConfirmTest"), new GalaxyRepositoryOptions(), new GalaxyReconnectOptions()) { AlarmAck = new GalaxyAlarmAckOptions { ConfirmTimeoutMs = confirmTimeoutMs }, }; return new GalaxyDriver( driverInstanceId: "drv-ack", options: options, hierarchySource: null, alarmAcknowledger: acknowledger, alarmFeed: feed, logger: logger is null ? null : new TypedLogger(logger), ackCredentialResolver: FixedAckCredentialResolver.Fallback()); } private static GalaxyAlarmTransition Transition(string alarmRef, GalaxyAlarmTransitionKind kind) => new( AlarmFullReference: alarmRef, SourceObjectReference: "Tank01", AlarmTypeName: "AnalogLimitAlarm.HiHi", TransitionKind: kind, SeverityBucket: AlarmSeverity.High, OpcUaSeverity: 700, RawMxAccessSeverity: 500, OriginalRaiseTimestampUtc: null, TransitionTimestampUtc: DateTime.UtcNow, OperatorUser: string.Empty, OperatorComment: string.Empty, Category: "Process", Description: "Tank 01 high-high level"); /// /// A fake gateway: every call answers with , mapped by the production /// . runs while the RPC is /// "in flight" (before the reply is evaluated). /// private sealed class ScriptedGateway(AcknowledgeAlarmReply reply) : IGalaxyAlarmAcknowledger { private int _calls; public Action? DuringCall { get; init; } public int Calls => Volatile.Read(ref _calls); public Task AcknowledgeAsync( string alarmFullReference, string comment, GalaxyAckCredential credential, CancellationToken cancellationToken) { Interlocked.Increment(ref _calls); DuringCall?.Invoke(); try { return Task.FromResult(GatewayGalaxyAlarmAcknowledger.EvaluateReply( reply, alarmFullReference, credential, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance)); } catch (Exception ex) { return Task.FromException(ex); } } } private sealed class FakeAlarmFeed : IGalaxyAlarmFeed { public event EventHandler? OnAlarmTransition; public void Start() { } public void Emit(GalaxyAlarmTransition transition) => OnAlarmTransition?.Invoke(this, transition); public ValueTask DisposeAsync() => ValueTask.CompletedTask; } private sealed class TypedLogger(CapturingLogger inner) : ILogger { private readonly Lock _gate = new(); public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(LogLevel logLevel) => true; public void Log(LogLevel l, EventId id, TState s, Exception? e, Func f) { lock (_gate) inner.Log(l, id, s, e, f); } } }