Recover WnWrap after local Alarmmgr exits

This commit is contained in:
Joseph Doherty
2026-11-21 13:51:12 -05:00
parent 3bb6182f91
commit c102f881b7
5 changed files with 88 additions and 35 deletions
@@ -804,7 +804,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
{ {
Mode = verifyFailover ? "Auto" : "ForceAlarmManager", Mode = verifyFailover ? "Auto" : "ForceAlarmManager",
ConsecutiveFailureThreshold = 3, ConsecutiveFailureThreshold = 3,
FailbackProbeIntervalSeconds = 1, FailbackProbeIntervalSeconds = outageDirectory is null ? 1 : 5,
FailbackStableProbes = 3, FailbackStableProbes = 3,
Discovery = new AlarmDiscoveryOptions Discovery = new AlarmDiscoveryOptions
{ {
@@ -841,12 +841,17 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
} }
} }
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest async Task<string> OpenDriverSessionAsync()
{ {
ClientSessionName = "rust-native-alarm-driver", OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false); {
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code); ClientSessionName = "rust-native-alarm-driver",
string sessionId = opened.SessionId; }, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
return opened.SessionId;
}
string sessionId = await OpenDriverSessionAsync().ConfigureAwait(false);
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken) async Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken)
{ {
@@ -857,6 +862,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
return reply; return reply;
} }
async Task<(int Item, int UserId)> BindDriverAsync(int server)
{
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
add.Command.AddItem.ItemDefinition = tag;
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
.ConfigureAwait(false)).AuthenticateUser.UserId;
Assert.NotEqual(0, userId);
return (item, userId);
}
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync( async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(
AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr) AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr)
{ {
@@ -905,13 +922,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
BulkReadResult original = Assert.Single(read.ReadBulk.Results); BulkReadResult original = Assert.Single(read.ReadBulk.Results);
Assert.True(original.WasSuccessful, original.ErrorMessage); Assert.True(original.WasSuccessful, original.ErrorMessage);
Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase); Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase);
MxCommandRequest add = CreateAddItemRequest(sessionId, server); (int item, int userId) = await BindDriverAsync(server).ConfigureAwait(false);
add.Command.AddItem.ItemDefinition = tag;
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
.ConfigureAwait(false)).AuthenticateUser.UserId;
Assert.NotEqual(0, userId);
MxCommandRequest WriteAlarm(bool active) MxCommandRequest WriteAlarm(bool active)
{ {
@@ -959,6 +970,23 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty); File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty);
} }
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false); await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
if (outageDirectory is not null)
{
// The recovery controller can restart the AppEngine and invalidate the writer's handles.
// The alarm feed stays open while the independent writer reconnects.
string ready = Path.Combine(outageDirectory, "ready-alarmmgr");
while (!File.Exists(ready))
{
await Task.Delay(100, token).ConfigureAwait(false);
}
await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
restore = null;
sessionId = await OpenDriverSessionAsync().ConfigureAwait(false);
server = (await InvokeAsync(CreateRegisterRequest(sessionId), token).ConfigureAwait(false)).Register.ServerHandle;
(item, userId) = await BindDriverAsync(server).ConfigureAwait(false);
restore = WriteAlarm(original.Value.BoolValue);
}
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false); await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false); await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise."); output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise.");
+5 -3
View File
@@ -146,9 +146,11 @@ For a real outage, build the normal worker and point
Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live
test options above. An external controller stops Alarmmgr when the test creates test options above. An external controller stops Alarmmgr when the test creates
`stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`. `stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`.
The controller should also restore Alarmmgr on timeout or failure. The test The controller creates `ready-alarmmgr` when the replacement is running and
uses live subtags while Alarmmgr is stopped and waits for a fresh Alarmmgr the test alarm's AppEngine is on scan. The controller should also restore
raise after recovery on the original stream. Alarmmgr on timeout or failure. The test uses live subtags while Alarmmgr is
stopped and waits for a fresh Alarmmgr raise after recovery on the original
stream.
## COM interface evidence ## COM interface evidence
+23 -12
View File
@@ -24,7 +24,7 @@ use crate::wire::mxaccess_gateway::v1::{
ProtocolStatus, ProtocolStatusCode as Code, QueryActiveAlarmsReplyPayload, ProtocolStatus, ProtocolStatusCode as Code, QueryActiveAlarmsReplyPayload,
SubscribeAlarmsCommand, SubscribeAlarmsCommand,
}; };
use local_alarmmgr::LocalAlarmmgr; use local_alarmmgr::{LocalAlarmmgr, Status as ManagerStatus};
use record::{parse_snapshot, Record, State}; use record::{parse_snapshot, Record, State};
use snapshot_worker::SnapshotProcess; use snapshot_worker::SnapshotProcess;
use subtag::Subtags; use subtag::Subtags;
@@ -108,11 +108,9 @@ impl Primary {
may_request: bool, may_request: bool,
now: Instant, now: Instant,
) -> Option<Result<(), HRESULT>> { ) -> Option<Result<(), HRESULT>> {
if self let manager = self.local_manager.as_mut().map(LocalAlarmmgr::observe);
.local_manager let manager_missing = manager == Some(ManagerStatus::Missing);
.as_mut() if manager == Some(ManagerStatus::Exited) || (emit && manager_missing) {
.is_some_and(|manager| !manager.available())
{
let lost_source = self.source.is_some(); let lost_source = self.source.is_some();
self.reset(); self.reset();
return (may_request || lost_source) return (may_request || lost_source)
@@ -129,8 +127,12 @@ impl Primary {
return match reply { return match reply {
Ok(reply) => { Ok(reply) => {
self.pending_until = None; self.pending_until = None;
let result = let result = reply.and_then(|xml| {
reply.and_then(|xml| self.apply_snapshot(proxy, limit, emit, &xml)); if manager_missing {
return Err((0x8007_0000 | ERROR_SERVICE_NOT_ACTIVE) as HRESULT);
}
self.apply_snapshot(proxy, limit, emit, &xml)
});
if result.is_err() { if result.is_err() {
self.reset(); self.reset();
} }
@@ -149,7 +151,8 @@ impl Primary {
} }
if may_request { if may_request {
if self.source.is_none() { let starting = self.source.is_none();
if starting {
self.source = match SnapshotProcess::start(&self.expression, limit) { self.source = match SnapshotProcess::start(&self.expression, limit) {
Ok(source) => Some(source), Ok(source) => Some(source),
Err(error) => { Err(error) => {
@@ -168,14 +171,22 @@ impl Primary {
self.reset(); self.reset();
return Some(Err(E_FAIL)); return Some(Err(E_FAIL));
} }
self.pending_until = Some(now + Duration::from_secs(5)); // A recovery probe can activate WnWrap while Alarmmgr is absent.
// Its initial request also has to register, subscribe and query.
let seconds = if starting && !emit { 12 } else { 5 };
self.pending_until = Some(now + Duration::from_secs(seconds));
} }
None None
} }
fn reset(&mut self) { fn reset(&mut self) {
self.pending_until = None; let idle = self.pending_until.take().is_none();
self.source = None; if let Some(source) = self.source.take() {
if idle {
// Let WnWrap release its registration and shared Alarmmgr user count.
source.finish();
}
}
} }
fn apply_snapshot( fn apply_snapshot(
@@ -16,6 +16,13 @@ pub enum LocalAlarmmgr {
Stopped, Stopped,
} }
#[derive(PartialEq, Eq)]
pub enum Status {
Available,
Exited,
Missing,
}
impl LocalAlarmmgr { impl LocalAlarmmgr {
pub fn for_subscription(expression: &str) -> Option<Self> { pub fn for_subscription(expression: &str) -> Option<Self> {
let (node, query) = expression.trim_start_matches('\\').split_once('\\')?; let (node, query) = expression.trim_start_matches('\\').split_once('\\')?;
@@ -29,20 +36,23 @@ impl LocalAlarmmgr {
.then_some(Self::Unobserved) .then_some(Self::Unobserved)
} }
pub fn available(&mut self) -> bool { pub fn observe(&mut self) -> Status {
if let Self::Running(process) = self { if let Self::Running(process) = self {
// SAFETY: this is an owned, waitable process handle. A zero timeout never blocks. // SAFETY: this is an owned, waitable process handle. A zero timeout never blocks.
if unsafe { WaitForSingleObject(process.as_raw_handle(), 0) } != WAIT_OBJECT_0 { if unsafe { WaitForSingleObject(process.as_raw_handle(), 0) } != WAIT_OBJECT_0 {
return true; return Status::Available;
} }
*self = Self::Stopped; *self = Self::Stopped;
return false; return Status::Exited;
} }
if let Some(process) = find_alarmmgr() { if let Some(process) = find_alarmmgr() {
*self = Self::Running(process); *self = Self::Running(process);
} }
// A process we could never observe does not override COM health. // A process we could never observe does not override COM health.
!matches!(self, Self::Stopped) match self {
Self::Stopped => Status::Missing,
_ => Status::Available,
}
} }
} }
@@ -14,7 +14,7 @@ use windows_sys::core::HRESULT;
use windows_sys::Win32::System::JobObjects::{ use windows_sys::Win32::System::JobObjects::{
AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK,
}; };
use windows_sys::Win32::System::Threading::{WaitForSingleObject, CREATE_NO_WINDOW}; use windows_sys::Win32::System::Threading::{WaitForSingleObject, CREATE_NO_WINDOW};
@@ -83,7 +83,7 @@ impl SnapshotProcess {
.name("wnwrap-snapshot-cleanup".into()) .name("wnwrap-snapshot-cleanup".into())
.spawn(move || { .spawn(move || {
// SAFETY: the child process handle is owned by this thread. // SAFETY: the child process handle is owned by this thread.
unsafe { WaitForSingleObject(child.as_raw_handle(), 250) }; unsafe { WaitForSingleObject(child.as_raw_handle(), 2_000) };
drop(job); drop(job);
}); });
} }
@@ -102,7 +102,9 @@ fn kill_on_close_job() -> io::Result<OwnedHandle> {
} }
let job = unsafe { OwnedHandle::from_raw_handle(raw) }; let job = unsafe { OwnedHandle::from_raw_handle(raw) };
let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; // The vendor may start Alarmmgr from the helper during registration.
limits.BasicLimitInformation.LimitFlags =
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK;
// SAFETY: the settings structure matches the job information class. // SAFETY: the settings structure matches the job information class.
if unsafe { if unsafe {
SetInformationJobObject( SetInformationJobObject(