Recover WnWrap after local Alarmmgr exits
This commit is contained in:
@@ -804,7 +804,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
{
|
||||
Mode = verifyFailover ? "Auto" : "ForceAlarmManager",
|
||||
ConsecutiveFailureThreshold = 3,
|
||||
FailbackProbeIntervalSeconds = 1,
|
||||
FailbackProbeIntervalSeconds = outageDirectory is null ? 1 : 5,
|
||||
FailbackStableProbes = 3,
|
||||
Discovery = new AlarmDiscoveryOptions
|
||||
{
|
||||
@@ -841,12 +841,17 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
}
|
||||
}
|
||||
|
||||
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
||||
async Task<string> OpenDriverSessionAsync()
|
||||
{
|
||||
ClientSessionName = "rust-native-alarm-driver",
|
||||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||||
string sessionId = opened.SessionId;
|
||||
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
||||
{
|
||||
ClientSessionName = "rust-native-alarm-driver",
|
||||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||||
return opened.SessionId;
|
||||
}
|
||||
|
||||
string sessionId = await OpenDriverSessionAsync().ConfigureAwait(false);
|
||||
|
||||
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -857,6 +862,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
return reply;
|
||||
}
|
||||
|
||||
async Task<(int Item, int UserId)> BindDriverAsync(int server)
|
||||
{
|
||||
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
|
||||
add.Command.AddItem.ItemDefinition = tag;
|
||||
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
|
||||
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
|
||||
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
|
||||
.ConfigureAwait(false)).AuthenticateUser.UserId;
|
||||
Assert.NotEqual(0, userId);
|
||||
return (item, userId);
|
||||
}
|
||||
|
||||
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(
|
||||
AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr)
|
||||
{
|
||||
@@ -905,13 +922,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
BulkReadResult original = Assert.Single(read.ReadBulk.Results);
|
||||
Assert.True(original.WasSuccessful, original.ErrorMessage);
|
||||
Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase);
|
||||
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
|
||||
add.Command.AddItem.ItemDefinition = tag;
|
||||
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
|
||||
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
|
||||
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
|
||||
.ConfigureAwait(false)).AuthenticateUser.UserId;
|
||||
Assert.NotEqual(0, userId);
|
||||
(int item, int userId) = await BindDriverAsync(server).ConfigureAwait(false);
|
||||
|
||||
MxCommandRequest WriteAlarm(bool active)
|
||||
{
|
||||
@@ -959,6 +970,23 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty);
|
||||
}
|
||||
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
|
||||
if (outageDirectory is not null)
|
||||
{
|
||||
// The recovery controller can restart the AppEngine and invalidate the writer's handles.
|
||||
// The alarm feed stays open while the independent writer reconnects.
|
||||
string ready = Path.Combine(outageDirectory, "ready-alarmmgr");
|
||||
while (!File.Exists(ready))
|
||||
{
|
||||
await Task.Delay(100, token).ConfigureAwait(false);
|
||||
}
|
||||
await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
|
||||
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
|
||||
restore = null;
|
||||
sessionId = await OpenDriverSessionAsync().ConfigureAwait(false);
|
||||
server = (await InvokeAsync(CreateRegisterRequest(sessionId), token).ConfigureAwait(false)).Register.ServerHandle;
|
||||
(item, userId) = await BindDriverAsync(server).ConfigureAwait(false);
|
||||
restore = WriteAlarm(original.Value.BoolValue);
|
||||
}
|
||||
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
|
||||
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
||||
output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise.");
|
||||
|
||||
@@ -146,9 +146,11 @@ For a real outage, build the normal worker and point
|
||||
Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live
|
||||
test options above. An external controller stops Alarmmgr when the test creates
|
||||
`stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`.
|
||||
The controller should also restore Alarmmgr on timeout or failure. The test
|
||||
uses live subtags while Alarmmgr is stopped and waits for a fresh Alarmmgr
|
||||
raise after recovery on the original stream.
|
||||
The controller creates `ready-alarmmgr` when the replacement is running and
|
||||
the test alarm's AppEngine is on scan. The controller should also restore
|
||||
Alarmmgr on timeout or failure. The test uses live subtags while Alarmmgr is
|
||||
stopped and waits for a fresh Alarmmgr raise after recovery on the original
|
||||
stream.
|
||||
|
||||
## COM interface evidence
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::wire::mxaccess_gateway::v1::{
|
||||
ProtocolStatus, ProtocolStatusCode as Code, QueryActiveAlarmsReplyPayload,
|
||||
SubscribeAlarmsCommand,
|
||||
};
|
||||
use local_alarmmgr::LocalAlarmmgr;
|
||||
use local_alarmmgr::{LocalAlarmmgr, Status as ManagerStatus};
|
||||
use record::{parse_snapshot, Record, State};
|
||||
use snapshot_worker::SnapshotProcess;
|
||||
use subtag::Subtags;
|
||||
@@ -108,11 +108,9 @@ impl Primary {
|
||||
may_request: bool,
|
||||
now: Instant,
|
||||
) -> Option<Result<(), HRESULT>> {
|
||||
if self
|
||||
.local_manager
|
||||
.as_mut()
|
||||
.is_some_and(|manager| !manager.available())
|
||||
{
|
||||
let manager = self.local_manager.as_mut().map(LocalAlarmmgr::observe);
|
||||
let manager_missing = manager == Some(ManagerStatus::Missing);
|
||||
if manager == Some(ManagerStatus::Exited) || (emit && manager_missing) {
|
||||
let lost_source = self.source.is_some();
|
||||
self.reset();
|
||||
return (may_request || lost_source)
|
||||
@@ -129,8 +127,12 @@ impl Primary {
|
||||
return match reply {
|
||||
Ok(reply) => {
|
||||
self.pending_until = None;
|
||||
let result =
|
||||
reply.and_then(|xml| self.apply_snapshot(proxy, limit, emit, &xml));
|
||||
let result = reply.and_then(|xml| {
|
||||
if manager_missing {
|
||||
return Err((0x8007_0000 | ERROR_SERVICE_NOT_ACTIVE) as HRESULT);
|
||||
}
|
||||
self.apply_snapshot(proxy, limit, emit, &xml)
|
||||
});
|
||||
if result.is_err() {
|
||||
self.reset();
|
||||
}
|
||||
@@ -149,7 +151,8 @@ impl Primary {
|
||||
}
|
||||
|
||||
if may_request {
|
||||
if self.source.is_none() {
|
||||
let starting = self.source.is_none();
|
||||
if starting {
|
||||
self.source = match SnapshotProcess::start(&self.expression, limit) {
|
||||
Ok(source) => Some(source),
|
||||
Err(error) => {
|
||||
@@ -168,14 +171,22 @@ impl Primary {
|
||||
self.reset();
|
||||
return Some(Err(E_FAIL));
|
||||
}
|
||||
self.pending_until = Some(now + Duration::from_secs(5));
|
||||
// A recovery probe can activate WnWrap while Alarmmgr is absent.
|
||||
// Its initial request also has to register, subscribe and query.
|
||||
let seconds = if starting && !emit { 12 } else { 5 };
|
||||
self.pending_until = Some(now + Duration::from_secs(seconds));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.pending_until = None;
|
||||
self.source = None;
|
||||
let idle = self.pending_until.take().is_none();
|
||||
if let Some(source) = self.source.take() {
|
||||
if idle {
|
||||
// Let WnWrap release its registration and shared Alarmmgr user count.
|
||||
source.finish();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_snapshot(
|
||||
|
||||
@@ -16,6 +16,13 @@ pub enum LocalAlarmmgr {
|
||||
Stopped,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub enum Status {
|
||||
Available,
|
||||
Exited,
|
||||
Missing,
|
||||
}
|
||||
|
||||
impl LocalAlarmmgr {
|
||||
pub fn for_subscription(expression: &str) -> Option<Self> {
|
||||
let (node, query) = expression.trim_start_matches('\\').split_once('\\')?;
|
||||
@@ -29,20 +36,23 @@ impl LocalAlarmmgr {
|
||||
.then_some(Self::Unobserved)
|
||||
}
|
||||
|
||||
pub fn available(&mut self) -> bool {
|
||||
pub fn observe(&mut self) -> Status {
|
||||
if let Self::Running(process) = self {
|
||||
// SAFETY: this is an owned, waitable process handle. A zero timeout never blocks.
|
||||
if unsafe { WaitForSingleObject(process.as_raw_handle(), 0) } != WAIT_OBJECT_0 {
|
||||
return true;
|
||||
return Status::Available;
|
||||
}
|
||||
*self = Self::Stopped;
|
||||
return false;
|
||||
return Status::Exited;
|
||||
}
|
||||
if let Some(process) = find_alarmmgr() {
|
||||
*self = Self::Running(process);
|
||||
}
|
||||
// A process we could never observe does not override COM health.
|
||||
!matches!(self, Self::Stopped)
|
||||
match self {
|
||||
Self::Stopped => Status::Missing,
|
||||
_ => Status::Available,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ use windows_sys::core::HRESULT;
|
||||
use windows_sys::Win32::System::JobObjects::{
|
||||
AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
|
||||
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
|
||||
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
|
||||
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK,
|
||||
};
|
||||
use windows_sys::Win32::System::Threading::{WaitForSingleObject, CREATE_NO_WINDOW};
|
||||
|
||||
@@ -83,7 +83,7 @@ impl SnapshotProcess {
|
||||
.name("wnwrap-snapshot-cleanup".into())
|
||||
.spawn(move || {
|
||||
// SAFETY: the child process handle is owned by this thread.
|
||||
unsafe { WaitForSingleObject(child.as_raw_handle(), 250) };
|
||||
unsafe { WaitForSingleObject(child.as_raw_handle(), 2_000) };
|
||||
drop(job);
|
||||
});
|
||||
}
|
||||
@@ -102,7 +102,9 @@ fn kill_on_close_job() -> io::Result<OwnedHandle> {
|
||||
}
|
||||
let job = unsafe { OwnedHandle::from_raw_handle(raw) };
|
||||
let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
|
||||
limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
|
||||
// The vendor may start Alarmmgr from the helper during registration.
|
||||
limits.BasicLimitInformation.LimitFlags =
|
||||
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK;
|
||||
// SAFETY: the settings structure matches the job information class.
|
||||
if unsafe {
|
||||
SetInformationJobObject(
|
||||
|
||||
Reference in New Issue
Block a user