Recover WnWrap after local Alarmmgr exits

This commit is contained in:
Joseph Doherty
2026-11-21 13:51:12 -05:00
parent 3bb6182f91
commit c102f881b7
5 changed files with 88 additions and 35 deletions
@@ -804,7 +804,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
{
Mode = verifyFailover ? "Auto" : "ForceAlarmManager",
ConsecutiveFailureThreshold = 3,
FailbackProbeIntervalSeconds = 1,
FailbackProbeIntervalSeconds = outageDirectory is null ? 1 : 5,
FailbackStableProbes = 3,
Discovery = new AlarmDiscoveryOptions
{
@@ -841,12 +841,17 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
}
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
async Task<string> OpenDriverSessionAsync()
{
ClientSessionName = "rust-native-alarm-driver",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string sessionId = opened.SessionId;
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-native-alarm-driver",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
return opened.SessionId;
}
string sessionId = await OpenDriverSessionAsync().ConfigureAwait(false);
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken)
{
@@ -857,6 +862,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
return reply;
}
async Task<(int Item, int UserId)> BindDriverAsync(int server)
{
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
add.Command.AddItem.ItemDefinition = tag;
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
.ConfigureAwait(false)).AuthenticateUser.UserId;
Assert.NotEqual(0, userId);
return (item, userId);
}
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(
AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr)
{
@@ -905,13 +922,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
BulkReadResult original = Assert.Single(read.ReadBulk.Results);
Assert.True(original.WasSuccessful, original.ErrorMessage);
Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase);
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
add.Command.AddItem.ItemDefinition = tag;
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
.ConfigureAwait(false)).AuthenticateUser.UserId;
Assert.NotEqual(0, userId);
(int item, int userId) = await BindDriverAsync(server).ConfigureAwait(false);
MxCommandRequest WriteAlarm(bool active)
{
@@ -959,6 +970,23 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty);
}
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
if (outageDirectory is not null)
{
// The recovery controller can restart the AppEngine and invalidate the writer's handles.
// The alarm feed stays open while the independent writer reconnects.
string ready = Path.Combine(outageDirectory, "ready-alarmmgr");
while (!File.Exists(ready))
{
await Task.Delay(100, token).ConfigureAwait(false);
}
await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
restore = null;
sessionId = await OpenDriverSessionAsync().ConfigureAwait(false);
server = (await InvokeAsync(CreateRegisterRequest(sessionId), token).ConfigureAwait(false)).Register.ServerHandle;
(item, userId) = await BindDriverAsync(server).ConfigureAwait(false);
restore = WriteAlarm(original.Value.BoolValue);
}
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise.");
+5 -3
View File
@@ -146,9 +146,11 @@ For a real outage, build the normal worker and point
Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live
test options above. An external controller stops Alarmmgr when the test creates
`stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`.
The controller should also restore Alarmmgr on timeout or failure. The test
uses live subtags while Alarmmgr is stopped and waits for a fresh Alarmmgr
raise after recovery on the original stream.
The controller creates `ready-alarmmgr` when the replacement is running and
the test alarm's AppEngine is on scan. The controller should also restore
Alarmmgr on timeout or failure. The test uses live subtags while Alarmmgr is
stopped and waits for a fresh Alarmmgr raise after recovery on the original
stream.
## COM interface evidence
+23 -12
View File
@@ -24,7 +24,7 @@ use crate::wire::mxaccess_gateway::v1::{
ProtocolStatus, ProtocolStatusCode as Code, QueryActiveAlarmsReplyPayload,
SubscribeAlarmsCommand,
};
use local_alarmmgr::LocalAlarmmgr;
use local_alarmmgr::{LocalAlarmmgr, Status as ManagerStatus};
use record::{parse_snapshot, Record, State};
use snapshot_worker::SnapshotProcess;
use subtag::Subtags;
@@ -108,11 +108,9 @@ impl Primary {
may_request: bool,
now: Instant,
) -> Option<Result<(), HRESULT>> {
if self
.local_manager
.as_mut()
.is_some_and(|manager| !manager.available())
{
let manager = self.local_manager.as_mut().map(LocalAlarmmgr::observe);
let manager_missing = manager == Some(ManagerStatus::Missing);
if manager == Some(ManagerStatus::Exited) || (emit && manager_missing) {
let lost_source = self.source.is_some();
self.reset();
return (may_request || lost_source)
@@ -129,8 +127,12 @@ impl Primary {
return match reply {
Ok(reply) => {
self.pending_until = None;
let result =
reply.and_then(|xml| self.apply_snapshot(proxy, limit, emit, &xml));
let result = reply.and_then(|xml| {
if manager_missing {
return Err((0x8007_0000 | ERROR_SERVICE_NOT_ACTIVE) as HRESULT);
}
self.apply_snapshot(proxy, limit, emit, &xml)
});
if result.is_err() {
self.reset();
}
@@ -149,7 +151,8 @@ impl Primary {
}
if may_request {
if self.source.is_none() {
let starting = self.source.is_none();
if starting {
self.source = match SnapshotProcess::start(&self.expression, limit) {
Ok(source) => Some(source),
Err(error) => {
@@ -168,14 +171,22 @@ impl Primary {
self.reset();
return Some(Err(E_FAIL));
}
self.pending_until = Some(now + Duration::from_secs(5));
// A recovery probe can activate WnWrap while Alarmmgr is absent.
// Its initial request also has to register, subscribe and query.
let seconds = if starting && !emit { 12 } else { 5 };
self.pending_until = Some(now + Duration::from_secs(seconds));
}
None
}
fn reset(&mut self) {
self.pending_until = None;
self.source = None;
let idle = self.pending_until.take().is_none();
if let Some(source) = self.source.take() {
if idle {
// Let WnWrap release its registration and shared Alarmmgr user count.
source.finish();
}
}
}
fn apply_snapshot(
@@ -16,6 +16,13 @@ pub enum LocalAlarmmgr {
Stopped,
}
#[derive(PartialEq, Eq)]
pub enum Status {
Available,
Exited,
Missing,
}
impl LocalAlarmmgr {
pub fn for_subscription(expression: &str) -> Option<Self> {
let (node, query) = expression.trim_start_matches('\\').split_once('\\')?;
@@ -29,20 +36,23 @@ impl LocalAlarmmgr {
.then_some(Self::Unobserved)
}
pub fn available(&mut self) -> bool {
pub fn observe(&mut self) -> Status {
if let Self::Running(process) = self {
// SAFETY: this is an owned, waitable process handle. A zero timeout never blocks.
if unsafe { WaitForSingleObject(process.as_raw_handle(), 0) } != WAIT_OBJECT_0 {
return true;
return Status::Available;
}
*self = Self::Stopped;
return false;
return Status::Exited;
}
if let Some(process) = find_alarmmgr() {
*self = Self::Running(process);
}
// A process we could never observe does not override COM health.
!matches!(self, Self::Stopped)
match self {
Self::Stopped => Status::Missing,
_ => Status::Available,
}
}
}
@@ -14,7 +14,7 @@ use windows_sys::core::HRESULT;
use windows_sys::Win32::System::JobObjects::{
AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK,
};
use windows_sys::Win32::System::Threading::{WaitForSingleObject, CREATE_NO_WINDOW};
@@ -83,7 +83,7 @@ impl SnapshotProcess {
.name("wnwrap-snapshot-cleanup".into())
.spawn(move || {
// SAFETY: the child process handle is owned by this thread.
unsafe { WaitForSingleObject(child.as_raw_handle(), 250) };
unsafe { WaitForSingleObject(child.as_raw_handle(), 2_000) };
drop(job);
});
}
@@ -102,7 +102,9 @@ fn kill_on_close_job() -> io::Result<OwnedHandle> {
}
let job = unsafe { OwnedHandle::from_raw_handle(raw) };
let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
// The vendor may start Alarmmgr from the helper during registration.
limits.BasicLimitInformation.LimitFlags =
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK;
// SAFETY: the settings structure matches the job information class.
if unsafe {
SetInformationJobObject(