Recover WnWrap after local Alarmmgr exits
This commit is contained in:
@@ -804,7 +804,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
{
|
{
|
||||||
Mode = verifyFailover ? "Auto" : "ForceAlarmManager",
|
Mode = verifyFailover ? "Auto" : "ForceAlarmManager",
|
||||||
ConsecutiveFailureThreshold = 3,
|
ConsecutiveFailureThreshold = 3,
|
||||||
FailbackProbeIntervalSeconds = 1,
|
FailbackProbeIntervalSeconds = outageDirectory is null ? 1 : 5,
|
||||||
FailbackStableProbes = 3,
|
FailbackStableProbes = 3,
|
||||||
Discovery = new AlarmDiscoveryOptions
|
Discovery = new AlarmDiscoveryOptions
|
||||||
{
|
{
|
||||||
@@ -841,12 +841,17 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
async Task<string> OpenDriverSessionAsync()
|
||||||
{
|
{
|
||||||
ClientSessionName = "rust-native-alarm-driver",
|
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
||||||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
{
|
||||||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
ClientSessionName = "rust-native-alarm-driver",
|
||||||
string sessionId = opened.SessionId;
|
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||||||
|
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||||||
|
return opened.SessionId;
|
||||||
|
}
|
||||||
|
|
||||||
|
string sessionId = await OpenDriverSessionAsync().ConfigureAwait(false);
|
||||||
|
|
||||||
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken)
|
async Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
@@ -857,6 +862,18 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
return reply;
|
return reply;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async Task<(int Item, int UserId)> BindDriverAsync(int server)
|
||||||
|
{
|
||||||
|
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
|
||||||
|
add.Command.AddItem.ItemDefinition = tag;
|
||||||
|
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
|
||||||
|
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
|
||||||
|
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
|
||||||
|
.ConfigureAwait(false)).AuthenticateUser.UserId;
|
||||||
|
Assert.NotEqual(0, userId);
|
||||||
|
return (item, userId);
|
||||||
|
}
|
||||||
|
|
||||||
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(
|
async Task<OnAlarmTransitionEvent> WaitForTransitionAsync(
|
||||||
AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr)
|
AlarmTransitionKind expected, string? ackComment = null, AlarmProviderMode provider = AlarmProviderMode.Alarmmgr)
|
||||||
{
|
{
|
||||||
@@ -905,13 +922,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
BulkReadResult original = Assert.Single(read.ReadBulk.Results);
|
BulkReadResult original = Assert.Single(read.ReadBulk.Results);
|
||||||
Assert.True(original.WasSuccessful, original.ErrorMessage);
|
Assert.True(original.WasSuccessful, original.ErrorMessage);
|
||||||
Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase);
|
Assert.Equal(MxValue.KindOneofCase.BoolValue, original.Value.KindCase);
|
||||||
MxCommandRequest add = CreateAddItemRequest(sessionId, server);
|
(int item, int userId) = await BindDriverAsync(server).ConfigureAwait(false);
|
||||||
add.Command.AddItem.ItemDefinition = tag;
|
|
||||||
int item = (await InvokeAsync(add, token).ConfigureAwait(false)).AddItem.ItemHandle;
|
|
||||||
await InvokeAsync(CreateAdviseRequest(sessionId, server, item), token).ConfigureAwait(false);
|
|
||||||
int userId = (await InvokeAsync(CreateAuthenticateUserRequest(sessionId, server, user, password), token)
|
|
||||||
.ConfigureAwait(false)).AuthenticateUser.UserId;
|
|
||||||
Assert.NotEqual(0, userId);
|
|
||||||
|
|
||||||
MxCommandRequest WriteAlarm(bool active)
|
MxCommandRequest WriteAlarm(bool active)
|
||||||
{
|
{
|
||||||
@@ -959,6 +970,23 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty);
|
File.WriteAllText(Path.Combine(outageDirectory!, "restore-alarmmgr"), string.Empty);
|
||||||
}
|
}
|
||||||
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
|
await WaitForModeAsync(AlarmProviderMode.Alarmmgr, "recovered").ConfigureAwait(false);
|
||||||
|
if (outageDirectory is not null)
|
||||||
|
{
|
||||||
|
// The recovery controller can restart the AppEngine and invalidate the writer's handles.
|
||||||
|
// The alarm feed stays open while the independent writer reconnects.
|
||||||
|
string ready = Path.Combine(outageDirectory, "ready-alarmmgr");
|
||||||
|
while (!File.Exists(ready))
|
||||||
|
{
|
||||||
|
await Task.Delay(100, token).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
await client.CloseSessionAsync(new CloseSessionRequest { SessionId = sessionId },
|
||||||
|
deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false);
|
||||||
|
restore = null;
|
||||||
|
sessionId = await OpenDriverSessionAsync().ConfigureAwait(false);
|
||||||
|
server = (await InvokeAsync(CreateRegisterRequest(sessionId), token).ConfigureAwait(false)).Register.ServerHandle;
|
||||||
|
(item, userId) = await BindDriverAsync(server).ConfigureAwait(false);
|
||||||
|
restore = WriteAlarm(original.Value.BoolValue);
|
||||||
|
}
|
||||||
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
|
await WriteConfirmedAsync(WriteAlarm(true), token).ConfigureAwait(false);
|
||||||
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
await WaitForTransitionAsync(AlarmTransitionKind.Raise).ConfigureAwait(false);
|
||||||
output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise.");
|
output.WriteLine("One StreamAlarms call delivered Alarmmgr Raise, Subtag Clear/Raise/Clear, and Alarmmgr Raise.");
|
||||||
|
|||||||
@@ -146,9 +146,11 @@ For a real outage, build the normal worker and point
|
|||||||
Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live
|
Run `GatewayGrpc_WithRustWorker_StreamsAcrossRealAlarmmgrOutage` with the live
|
||||||
test options above. An external controller stops Alarmmgr when the test creates
|
test options above. An external controller stops Alarmmgr when the test creates
|
||||||
`stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`.
|
`stop-alarmmgr` and restores it when the test creates `restore-alarmmgr`.
|
||||||
The controller should also restore Alarmmgr on timeout or failure. The test
|
The controller creates `ready-alarmmgr` when the replacement is running and
|
||||||
uses live subtags while Alarmmgr is stopped and waits for a fresh Alarmmgr
|
the test alarm's AppEngine is on scan. The controller should also restore
|
||||||
raise after recovery on the original stream.
|
Alarmmgr on timeout or failure. The test uses live subtags while Alarmmgr is
|
||||||
|
stopped and waits for a fresh Alarmmgr raise after recovery on the original
|
||||||
|
stream.
|
||||||
|
|
||||||
## COM interface evidence
|
## COM interface evidence
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ use crate::wire::mxaccess_gateway::v1::{
|
|||||||
ProtocolStatus, ProtocolStatusCode as Code, QueryActiveAlarmsReplyPayload,
|
ProtocolStatus, ProtocolStatusCode as Code, QueryActiveAlarmsReplyPayload,
|
||||||
SubscribeAlarmsCommand,
|
SubscribeAlarmsCommand,
|
||||||
};
|
};
|
||||||
use local_alarmmgr::LocalAlarmmgr;
|
use local_alarmmgr::{LocalAlarmmgr, Status as ManagerStatus};
|
||||||
use record::{parse_snapshot, Record, State};
|
use record::{parse_snapshot, Record, State};
|
||||||
use snapshot_worker::SnapshotProcess;
|
use snapshot_worker::SnapshotProcess;
|
||||||
use subtag::Subtags;
|
use subtag::Subtags;
|
||||||
@@ -108,11 +108,9 @@ impl Primary {
|
|||||||
may_request: bool,
|
may_request: bool,
|
||||||
now: Instant,
|
now: Instant,
|
||||||
) -> Option<Result<(), HRESULT>> {
|
) -> Option<Result<(), HRESULT>> {
|
||||||
if self
|
let manager = self.local_manager.as_mut().map(LocalAlarmmgr::observe);
|
||||||
.local_manager
|
let manager_missing = manager == Some(ManagerStatus::Missing);
|
||||||
.as_mut()
|
if manager == Some(ManagerStatus::Exited) || (emit && manager_missing) {
|
||||||
.is_some_and(|manager| !manager.available())
|
|
||||||
{
|
|
||||||
let lost_source = self.source.is_some();
|
let lost_source = self.source.is_some();
|
||||||
self.reset();
|
self.reset();
|
||||||
return (may_request || lost_source)
|
return (may_request || lost_source)
|
||||||
@@ -129,8 +127,12 @@ impl Primary {
|
|||||||
return match reply {
|
return match reply {
|
||||||
Ok(reply) => {
|
Ok(reply) => {
|
||||||
self.pending_until = None;
|
self.pending_until = None;
|
||||||
let result =
|
let result = reply.and_then(|xml| {
|
||||||
reply.and_then(|xml| self.apply_snapshot(proxy, limit, emit, &xml));
|
if manager_missing {
|
||||||
|
return Err((0x8007_0000 | ERROR_SERVICE_NOT_ACTIVE) as HRESULT);
|
||||||
|
}
|
||||||
|
self.apply_snapshot(proxy, limit, emit, &xml)
|
||||||
|
});
|
||||||
if result.is_err() {
|
if result.is_err() {
|
||||||
self.reset();
|
self.reset();
|
||||||
}
|
}
|
||||||
@@ -149,7 +151,8 @@ impl Primary {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if may_request {
|
if may_request {
|
||||||
if self.source.is_none() {
|
let starting = self.source.is_none();
|
||||||
|
if starting {
|
||||||
self.source = match SnapshotProcess::start(&self.expression, limit) {
|
self.source = match SnapshotProcess::start(&self.expression, limit) {
|
||||||
Ok(source) => Some(source),
|
Ok(source) => Some(source),
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
@@ -168,14 +171,22 @@ impl Primary {
|
|||||||
self.reset();
|
self.reset();
|
||||||
return Some(Err(E_FAIL));
|
return Some(Err(E_FAIL));
|
||||||
}
|
}
|
||||||
self.pending_until = Some(now + Duration::from_secs(5));
|
// A recovery probe can activate WnWrap while Alarmmgr is absent.
|
||||||
|
// Its initial request also has to register, subscribe and query.
|
||||||
|
let seconds = if starting && !emit { 12 } else { 5 };
|
||||||
|
self.pending_until = Some(now + Duration::from_secs(seconds));
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reset(&mut self) {
|
fn reset(&mut self) {
|
||||||
self.pending_until = None;
|
let idle = self.pending_until.take().is_none();
|
||||||
self.source = None;
|
if let Some(source) = self.source.take() {
|
||||||
|
if idle {
|
||||||
|
// Let WnWrap release its registration and shared Alarmmgr user count.
|
||||||
|
source.finish();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn apply_snapshot(
|
fn apply_snapshot(
|
||||||
|
|||||||
@@ -16,6 +16,13 @@ pub enum LocalAlarmmgr {
|
|||||||
Stopped,
|
Stopped,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Eq)]
|
||||||
|
pub enum Status {
|
||||||
|
Available,
|
||||||
|
Exited,
|
||||||
|
Missing,
|
||||||
|
}
|
||||||
|
|
||||||
impl LocalAlarmmgr {
|
impl LocalAlarmmgr {
|
||||||
pub fn for_subscription(expression: &str) -> Option<Self> {
|
pub fn for_subscription(expression: &str) -> Option<Self> {
|
||||||
let (node, query) = expression.trim_start_matches('\\').split_once('\\')?;
|
let (node, query) = expression.trim_start_matches('\\').split_once('\\')?;
|
||||||
@@ -29,20 +36,23 @@ impl LocalAlarmmgr {
|
|||||||
.then_some(Self::Unobserved)
|
.then_some(Self::Unobserved)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn available(&mut self) -> bool {
|
pub fn observe(&mut self) -> Status {
|
||||||
if let Self::Running(process) = self {
|
if let Self::Running(process) = self {
|
||||||
// SAFETY: this is an owned, waitable process handle. A zero timeout never blocks.
|
// SAFETY: this is an owned, waitable process handle. A zero timeout never blocks.
|
||||||
if unsafe { WaitForSingleObject(process.as_raw_handle(), 0) } != WAIT_OBJECT_0 {
|
if unsafe { WaitForSingleObject(process.as_raw_handle(), 0) } != WAIT_OBJECT_0 {
|
||||||
return true;
|
return Status::Available;
|
||||||
}
|
}
|
||||||
*self = Self::Stopped;
|
*self = Self::Stopped;
|
||||||
return false;
|
return Status::Exited;
|
||||||
}
|
}
|
||||||
if let Some(process) = find_alarmmgr() {
|
if let Some(process) = find_alarmmgr() {
|
||||||
*self = Self::Running(process);
|
*self = Self::Running(process);
|
||||||
}
|
}
|
||||||
// A process we could never observe does not override COM health.
|
// A process we could never observe does not override COM health.
|
||||||
!matches!(self, Self::Stopped)
|
match self {
|
||||||
|
Self::Stopped => Status::Missing,
|
||||||
|
_ => Status::Available,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ use windows_sys::core::HRESULT;
|
|||||||
use windows_sys::Win32::System::JobObjects::{
|
use windows_sys::Win32::System::JobObjects::{
|
||||||
AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
|
AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
|
||||||
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
|
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
|
||||||
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
|
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK,
|
||||||
};
|
};
|
||||||
use windows_sys::Win32::System::Threading::{WaitForSingleObject, CREATE_NO_WINDOW};
|
use windows_sys::Win32::System::Threading::{WaitForSingleObject, CREATE_NO_WINDOW};
|
||||||
|
|
||||||
@@ -83,7 +83,7 @@ impl SnapshotProcess {
|
|||||||
.name("wnwrap-snapshot-cleanup".into())
|
.name("wnwrap-snapshot-cleanup".into())
|
||||||
.spawn(move || {
|
.spawn(move || {
|
||||||
// SAFETY: the child process handle is owned by this thread.
|
// SAFETY: the child process handle is owned by this thread.
|
||||||
unsafe { WaitForSingleObject(child.as_raw_handle(), 250) };
|
unsafe { WaitForSingleObject(child.as_raw_handle(), 2_000) };
|
||||||
drop(job);
|
drop(job);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -102,7 +102,9 @@ fn kill_on_close_job() -> io::Result<OwnedHandle> {
|
|||||||
}
|
}
|
||||||
let job = unsafe { OwnedHandle::from_raw_handle(raw) };
|
let job = unsafe { OwnedHandle::from_raw_handle(raw) };
|
||||||
let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
|
let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
|
||||||
limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
|
// The vendor may start Alarmmgr from the helper during registration.
|
||||||
|
limits.BasicLimitInformation.LimitFlags =
|
||||||
|
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK;
|
||||||
// SAFETY: the settings structure matches the job information class.
|
// SAFETY: the settings structure matches the job information class.
|
||||||
if unsafe {
|
if unsafe {
|
||||||
SetInformationJobObject(
|
SetInformationJobObject(
|
||||||
|
|||||||
Reference in New Issue
Block a user