From c102f881b7bc49360de7c37e4a01cc42f40dd3b5 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 21 Nov 2026 13:51:12 -0500 Subject: [PATCH] Recover WnWrap after local Alarmmgr exits --- .../WorkerLiveMxAccessSmokeTests.cs | 54 ++++++++++++++----- src/mxgateway-worker-rust/README.md | 8 +-- src/mxgateway-worker-rust/src/alarm.rs | 35 +++++++----- .../src/alarm/local_alarmmgr.rs | 18 +++++-- .../src/alarm/snapshot_worker.rs | 8 +-- 5 files changed, 88 insertions(+), 35 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs b/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs index 8a92674f..80ca3d66 100644 --- a/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs +++ b/src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs @@ -804,7 +804,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) { Mode = verifyFailover ? "Auto" : "ForceAlarmManager", ConsecutiveFailureThreshold = 3, - FailbackProbeIntervalSeconds = 1, + FailbackProbeIntervalSeconds = outageDirectory is null ? 1 : 5, FailbackStableProbes = 3, Discovery = new AlarmDiscoveryOptions { @@ -841,12 +841,17 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) } } - OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest + async Task OpenDriverSessionAsync() { - ClientSessionName = "rust-native-alarm-driver", - }, cancellationToken: token).ResponseAsync.ConfigureAwait(false); - Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code); - string sessionId = opened.SessionId; + OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest + { + ClientSessionName = "rust-native-alarm-driver", + }, cancellationToken: token).ResponseAsync.ConfigureAwait(false); + Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code); + return opened.SessionId; + } + + string sessionId = await OpenDriverSessionAsync().ConfigureAwait(false); async Task InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken) { @@ -857,6 +862,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) return reply; } + async Task<(int Item, int UserId)> BindDriverAsync(int server) + { + MxCommandRequest add = CreateAddItemRequest(sessionId, server); + add.Command.AddItem.ItemDefinition = tag; + int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle; + await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false); + int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token) + .ConfigureAwait(false)).AuthenticateUser.UserId; + Assert.NotEqual(0, userId); + return (item, userId); + } + async Task WaitForTransitionAsync( AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr) { @@ -905,13 +922,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) BulkReadResult original = Assert.Single(read.ReadBulk.Results); Assert.True(original.WasSuccessful, original.ErrorMessage); Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase); - MxCommandRequest add = CreateAddItemRequest(sessionId, server); - add.Command.AddItem.ItemDefinition = tag; - int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle; - await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false); - int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token) - .ConfigureAwait(false)).AuthenticateUser.UserId; - Assert.NotEqual(0, userId); + (int item, int userId) = await BindDriverAsync(server).ConfigureAwait(false); MxCommandRequest WriteAlarm(bool active) { @@ -959,6 +970,23 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output) File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty); } await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false); + if (outageDirectory is not null) + { + // The recovery controller can restart the AppEngine and invalidate the writer's handles. + // The alarm feed stays open while the independent writer reconnects. + string ready = Path.Combine(outageDirectory, "ready-alarmmgr"); + while (!File.Exists(ready)) + { + await Task.Delay(100, token).ConfigureAwait(false); + } + await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId }, + deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false); + restore = null; + sessionId = await OpenDriverSessionAsync().ConfigureAwait(false); + server = (await InvokeAsync(CreateRegisterRequest(sessionId), token).ConfigureAwait(false)).Register.ServerHandle; + (item, userId) = await BindDriverAsync(server).ConfigureAwait(false); + restore = WriteAlarm(original.Value.BoolValue); + } await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false); await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false); output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise."); diff --git a/src/mxgateway-worker-rust/README.md b/src/mxgateway-worker-rust/README.md index 61641e0e..001eabb8 100644 --- a/src/mxgateway-worker-rust/README.md +++ b/src/mxgateway-worker-rust/README.md @@ -146,9 +146,11 @@ For a real outage, build the normal worker and point Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live test options above. An external controller stops Alarmmgr when the test creates `stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`. -The controller should also restore Alarmmgr on timeout or failure. The test -uses live subtags while Alarmmgr is stopped and waits for a fresh Alarmmgr -raise after recovery on the original stream. +The controller creates `ready-alarmmgr` when the replacement is running and +the test alarm's AppEngine is on scan. The controller should also restore +Alarmmgr on timeout or failure. The test uses live subtags while Alarmmgr is +stopped and waits for a fresh Alarmmgr raise after recovery on the original +stream. ## COM interface evidence diff --git a/src/mxgateway-worker-rust/src/alarm.rs b/src/mxgateway-worker-rust/src/alarm.rs index ae6bc60d..1ec7c91e 100644 --- a/src/mxgateway-worker-rust/src/alarm.rs +++ b/src/mxgateway-worker-rust/src/alarm.rs @@ -24,7 +24,7 @@ use crate::wire::mxaccess_gateway::v1::{ ProtocolStatus, ProtocolStatusCode as Code, QueryActiveAlarmsReplyPayload, SubscribeAlarmsCommand, }; -use local_alarmmgr::LocalAlarmmgr; +use local_alarmmgr::{LocalAlarmmgr, Status as ManagerStatus}; use record::{parse_snapshot, Record, State}; use snapshot_worker::SnapshotProcess; use subtag::Subtags; @@ -108,11 +108,9 @@ impl Primary { may_request: bool, now: Instant, ) -> Option> { - if self - .local_manager - .as_mut() - .is_some_and(|manager| !manager.available()) - { + let manager = self.local_manager.as_mut().map(LocalAlarmmgr::observe); + let manager_missing = manager == Some(ManagerStatus::Missing); + if manager == Some(ManagerStatus::Exited) || (emit && manager_missing) { let lost_source = self.source.is_some(); self.reset(); return (may_request || lost_source) @@ -129,8 +127,12 @@ impl Primary { return match reply { Ok(reply) => { self.pending_until = None; - let result = - reply.and_then(|xml| self.apply_snapshot(proxy, limit, emit, &xml)); + let result = reply.and_then(|xml| { + if manager_missing { + return Err((0x8007_0000 | ERROR_SERVICE_NOT_ACTIVE) as HRESULT); + } + self.apply_snapshot(proxy, limit, emit, &xml) + }); if result.is_err() { self.reset(); } @@ -149,7 +151,8 @@ impl Primary { } if may_request { - if self.source.is_none() { + let starting = self.source.is_none(); + if starting { self.source = match SnapshotProcess::start(&self.expression, limit) { Ok(source) => Some(source), Err(error) => { @@ -168,14 +171,22 @@ impl Primary { self.reset(); return Some(Err(E_FAIL)); } - self.pending_until = Some(now + Duration::from_secs(5)); + // A recovery probe can activate WnWrap while Alarmmgr is absent. + // Its initial request also has to register, subscribe and query. + let seconds = if starting && !emit { 12 } else { 5 }; + self.pending_until = Some(now + Duration::from_secs(seconds)); } None } fn reset(&mut self) { - self.pending_until = None; - self.source = None; + let idle = self.pending_until.take().is_none(); + if let Some(source) = self.source.take() { + if idle { + // Let WnWrap release its registration and shared Alarmmgr user count. + source.finish(); + } + } } fn apply_snapshot( diff --git a/src/mxgateway-worker-rust/src/alarm/local_alarmmgr.rs b/src/mxgateway-worker-rust/src/alarm/local_alarmmgr.rs index b763b42e..798314ba 100644 --- a/src/mxgateway-worker-rust/src/alarm/local_alarmmgr.rs +++ b/src/mxgateway-worker-rust/src/alarm/local_alarmmgr.rs @@ -16,6 +16,13 @@ pub enum LocalAlarmmgr { Stopped, } +#[derive(PartialEq, Eq)] +pub enum Status { + Available, + Exited, + Missing, +} + impl LocalAlarmmgr { pub fn for_subscription(expression: &str) -> Option { let (node, query) = expression.trim_start_matches('\\').split_once('\\')?; @@ -29,20 +36,23 @@ impl LocalAlarmmgr { .then_some(Self::Unobserved) } - pub fn available(&mut self) -> bool { + pub fn observe(&mut self) -> Status { if let Self::Running(process) = self { // SAFETY: this is an owned, waitable process handle. A zero timeout never blocks. if unsafe { WaitForSingleObject(process.as_raw_handle(), 0) } != WAIT_OBJECT_0 { - return true; + return Status::Available; } *self = Self::Stopped; - return false; + return Status::Exited; } if let Some(process) = find_alarmmgr() { *self = Self::Running(process); } // A process we could never observe does not override COM health. - !matches!(self, Self::Stopped) + match self { + Self::Stopped => Status::Missing, + _ => Status::Available, + } } } diff --git a/src/mxgateway-worker-rust/src/alarm/snapshot_worker.rs b/src/mxgateway-worker-rust/src/alarm/snapshot_worker.rs index 4b0ced93..619c335f 100644 --- a/src/mxgateway-worker-rust/src/alarm/snapshot_worker.rs +++ b/src/mxgateway-worker-rust/src/alarm/snapshot_worker.rs @@ -14,7 +14,7 @@ use windows_sys::core::HRESULT; use windows_sys::Win32::System::JobObjects::{ AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, - JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK, }; use windows_sys::Win32::System::Threading::{WaitForSingleObject, CREATE_NO_WINDOW}; @@ -83,7 +83,7 @@ impl SnapshotProcess { .name("wnwrap-snapshot-cleanup".into()) .spawn(move || { // SAFETY: the child process handle is owned by this thread. - unsafe { WaitForSingleObject(child.as_raw_handle(), 250) }; + unsafe { WaitForSingleObject(child.as_raw_handle(), 2_000) }; drop(job); }); } @@ -102,7 +102,9 @@ fn kill_on_close_job() -> io::Result { } let job = unsafe { OwnedHandle::from_raw_handle(raw) }; let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); - limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + // The vendor may start Alarmmgr from the helper during registration. + limits.BasicLimitInformation.LimitFlags = + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK; // SAFETY: the settings structure matches the job information class. if unsafe { SetInformationJobObject(