Isolate WnWrap recovery from the MXAccess STA
This commit is contained in:
@@ -941,7 +941,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
|||||||
if (verifyFailover)
|
if (verifyFailover)
|
||||||
{
|
{
|
||||||
string stopRequest = faultMarker ?? Path.Combine(outageDirectory!, "stop-alarmmgr");
|
string stopRequest = faultMarker ?? Path.Combine(outageDirectory!, "stop-alarmmgr");
|
||||||
File.WriteAllText(stopRequest, string.Empty);
|
File.WriteAllText(stopRequest, faultMarker is null ? string.Empty : "stall");
|
||||||
await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false);
|
await WaitForModeAsync(AlarmProviderMode.Subtag, "primary poll failed").ConfigureAwait(false);
|
||||||
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
|
await WriteConfirmedAsync(WriteAlarm(false), token).ConfigureAwait(false);
|
||||||
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
|
await WaitForTransitionAsync(AlarmTransitionKind.Clear, provider: AlarmProviderMode.Subtag).ConfigureAwait(false);
|
||||||
|
|||||||
@@ -22,7 +22,10 @@ windows-sys = { version = "0.61", features = [
|
|||||||
"Win32_Globalization",
|
"Win32_Globalization",
|
||||||
"Win32_Security",
|
"Win32_Security",
|
||||||
"Win32_System_Com",
|
"Win32_System_Com",
|
||||||
|
"Win32_System_Diagnostics_ToolHelp",
|
||||||
|
"Win32_System_JobObjects",
|
||||||
"Win32_System_Ole",
|
"Win32_System_Ole",
|
||||||
|
"Win32_System_RemoteDesktop",
|
||||||
"Win32_System_Threading",
|
"Win32_System_Threading",
|
||||||
"Win32_System_Time",
|
"Win32_System_Time",
|
||||||
"Win32_System_Variant",
|
"Win32_System_Variant",
|
||||||
|
|||||||
@@ -43,9 +43,15 @@ stream.
|
|||||||
The Rust worker handles alarm subscription, active snapshots, standby watch-list
|
The Rust worker handles alarm subscription, active snapshots, standby watch-list
|
||||||
replacement, and authenticated acknowledgement in the worker process. It uses
|
replacement, and authenticated acknowledgement in the worker process. It uses
|
||||||
the installed 32-bit AVEVA WnWrap COM alarm consumer for Alarmmgr and MXAccess
|
the installed 32-bit AVEVA WnWrap COM alarm consumer for Alarmmgr and MXAccess
|
||||||
advice for the optional subtag fallback. The worker polls WnWrap on its COM
|
advice for the optional subtag fallback. The Rust executable launches itself in
|
||||||
thread, emits changes on the existing event queue, and retains previously seen
|
a helper mode with a dedicated COM thread for WnWrap registration and polling.
|
||||||
records that a capped snapshot may have omitted.
|
Each snapshot request has a five-second deadline. If WnWrap stalls or fails,
|
||||||
|
the worker discards the helper and starts a fresh one for the next attempt,
|
||||||
|
while the MXAccess STA keeps serving commands and subtag subscriptions.
|
||||||
|
For a local Galaxy subscription with automatic fallback, the worker also watches
|
||||||
|
the Session 0 Alarmmgr process once it can open a handle to it. It waits for a
|
||||||
|
replacement process before probing and requires consecutive successful snapshots
|
||||||
|
before switching back. Snapshots retain records a capped fetch may have omitted.
|
||||||
|
|
||||||
To acknowledge a Galaxy alarm, the worker authenticates the operator with
|
To acknowledge a Galaxy alarm, the worker authenticates the operator with
|
||||||
MXAccess, writes the comment to the alarm's `AckMsg` attribute, and waits for
|
MXAccess, writes the comment to the alarm's `AckMsg` attribute, and waits for
|
||||||
@@ -129,8 +135,8 @@ $env:MXGATEWAY_LIVE_MXACCESS_WORKER_EXE = (Resolve-Path 'src/mxgateway-worker-ru
|
|||||||
dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests --filter 'FullyQualifiedName~StreamsAcrossAutomaticAlarmFailoverAndFailback'
|
dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests --filter 'FullyQualifiedName~StreamsAcrossAutomaticAlarmFailoverAndFailback'
|
||||||
```
|
```
|
||||||
|
|
||||||
The test temporarily makes the launched worker's Alarmmgr snapshot fetches
|
The test stalls the Rust helper's Alarmmgr snapshot fetches past the worker's
|
||||||
return an error. It drives the same development alarm through
|
request deadline. It drives the same development alarm through
|
||||||
`Alarmmgr → Subtag → Alarmmgr` and checks both provider statuses and subsequent
|
`Alarmmgr → Subtag → Alarmmgr` and checks both provider statuses and subsequent
|
||||||
transitions on one open `StreamAlarms` call. The fault injection is compiled only
|
transitions on one open `StreamAlarms` call. The fault injection is compiled only
|
||||||
when both debug assertions and the Cargo feature are enabled.
|
when both debug assertions and the Cargo feature are enabled.
|
||||||
|
|||||||
@@ -1,14 +1,18 @@
|
|||||||
mod ack;
|
mod ack;
|
||||||
|
mod local_alarmmgr;
|
||||||
mod record;
|
mod record;
|
||||||
|
pub(crate) mod snapshot_worker;
|
||||||
mod subtag;
|
mod subtag;
|
||||||
mod wnwrap;
|
mod wnwrap;
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::mpsc::TryRecvError;
|
||||||
use std::time::{Duration, Instant, SystemTime};
|
use std::time::{Duration, Instant, SystemTime};
|
||||||
|
|
||||||
|
use tokio::sync::oneshot;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use windows_sys::core::HRESULT;
|
use windows_sys::core::HRESULT;
|
||||||
use windows_sys::Win32::Foundation::E_FAIL;
|
use windows_sys::Win32::Foundation::{ERROR_SERVICE_NOT_ACTIVE, ERROR_TIMEOUT, E_FAIL};
|
||||||
use windows_sys::Win32::System::Threading::INFINITE;
|
use windows_sys::Win32::System::Threading::INFINITE;
|
||||||
|
|
||||||
use crate::mxaccess::{CommandError, CommandResult, Outcome, Proxy};
|
use crate::mxaccess::{CommandError, CommandResult, Outcome, Proxy};
|
||||||
@@ -20,9 +24,10 @@ use crate::wire::mxaccess_gateway::v1::{
|
|||||||
ProtocolStatus, ProtocolStatusCode as Code, QueryActiveAlarmsReplyPayload,
|
ProtocolStatus, ProtocolStatusCode as Code, QueryActiveAlarmsReplyPayload,
|
||||||
SubscribeAlarmsCommand,
|
SubscribeAlarmsCommand,
|
||||||
};
|
};
|
||||||
|
use local_alarmmgr::LocalAlarmmgr;
|
||||||
use record::{parse_snapshot, Record, State};
|
use record::{parse_snapshot, Record, State};
|
||||||
|
use snapshot_worker::SnapshotProcess;
|
||||||
use subtag::Subtags;
|
use subtag::Subtags;
|
||||||
use wnwrap::WnWrap;
|
|
||||||
|
|
||||||
pub fn handles(kind: Kind) -> bool {
|
pub fn handles(kind: Kind) -> bool {
|
||||||
matches!(
|
matches!(
|
||||||
@@ -87,21 +92,103 @@ fn handle(result: Outcome, step: &str) -> Result<i32, AlarmError> {
|
|||||||
|
|
||||||
struct Primary {
|
struct Primary {
|
||||||
expression: String,
|
expression: String,
|
||||||
consumer: Option<WnWrap>,
|
source: Option<SnapshotProcess>,
|
||||||
|
local_manager: Option<LocalAlarmmgr>,
|
||||||
|
pending_until: Option<Instant>,
|
||||||
records: BTreeMap<Uuid, Record>,
|
records: BTreeMap<Uuid, Record>,
|
||||||
truncated: bool,
|
truncated: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Primary {
|
impl Primary {
|
||||||
fn poll(&mut self, proxy: &Proxy, limit: i32, emit: bool) -> Result<(), HRESULT> {
|
fn poll(
|
||||||
if self.consumer.is_none() {
|
&mut self,
|
||||||
self.consumer = Some(WnWrap::subscribe(&self.expression)?);
|
proxy: &Proxy,
|
||||||
|
limit: i32,
|
||||||
|
emit: bool,
|
||||||
|
may_request: bool,
|
||||||
|
now: Instant,
|
||||||
|
) -> Option<Result<(), HRESULT>> {
|
||||||
|
if self
|
||||||
|
.local_manager
|
||||||
|
.as_mut()
|
||||||
|
.is_some_and(|manager| !manager.available())
|
||||||
|
{
|
||||||
|
let lost_source = self.source.is_some();
|
||||||
|
self.reset();
|
||||||
|
return (may_request || lost_source)
|
||||||
|
.then_some(Err((0x8007_0000 | ERROR_SERVICE_NOT_ACTIVE) as HRESULT));
|
||||||
}
|
}
|
||||||
let xml = self.consumer.as_ref().unwrap().snapshot(limit)?;
|
|
||||||
if xml.is_empty() {
|
if let Some(deadline) = self.pending_until {
|
||||||
return Ok(());
|
let reply = self
|
||||||
|
.source
|
||||||
|
.as_ref()
|
||||||
|
.expect("request has a helper")
|
||||||
|
.replies
|
||||||
|
.try_recv();
|
||||||
|
return match reply {
|
||||||
|
Ok(reply) => {
|
||||||
|
self.pending_until = None;
|
||||||
|
let result =
|
||||||
|
reply.and_then(|xml| self.apply_snapshot(proxy, limit, emit, &xml));
|
||||||
|
if result.is_err() {
|
||||||
|
self.reset();
|
||||||
}
|
}
|
||||||
let (records, count) = parse_snapshot(&xml).map_err(|_| E_FAIL)?;
|
Some(result)
|
||||||
|
}
|
||||||
|
Err(TryRecvError::Empty) if now < deadline => None,
|
||||||
|
Err(error) => {
|
||||||
|
self.reset();
|
||||||
|
Some(Err(if error == TryRecvError::Empty {
|
||||||
|
(0x8007_0000 | ERROR_TIMEOUT) as HRESULT
|
||||||
|
} else {
|
||||||
|
E_FAIL
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if may_request {
|
||||||
|
if self.source.is_none() {
|
||||||
|
self.source = match SnapshotProcess::start(&self.expression, limit) {
|
||||||
|
Ok(source) => Some(source),
|
||||||
|
Err(error) => {
|
||||||
|
eprintln!("Failed to start WnWrap snapshot worker: {error}");
|
||||||
|
return Some(Err(E_FAIL));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if self
|
||||||
|
.source
|
||||||
|
.as_mut()
|
||||||
|
.expect("started helper")
|
||||||
|
.request()
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
self.reset();
|
||||||
|
return Some(Err(E_FAIL));
|
||||||
|
}
|
||||||
|
self.pending_until = Some(now + Duration::from_secs(5));
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset(&mut self) {
|
||||||
|
self.pending_until = None;
|
||||||
|
self.source = None;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_snapshot(
|
||||||
|
&mut self,
|
||||||
|
proxy: &Proxy,
|
||||||
|
limit: i32,
|
||||||
|
emit: bool,
|
||||||
|
xml: &str,
|
||||||
|
) -> Result<(), HRESULT> {
|
||||||
|
if xml.trim().is_empty() {
|
||||||
|
return if emit { Ok(()) } else { Err(E_FAIL) };
|
||||||
|
}
|
||||||
|
let (records, count) = parse_snapshot(xml).ok_or(E_FAIL)?;
|
||||||
let previous = std::mem::replace(&mut self.records, records);
|
let previous = std::mem::replace(&mut self.records, records);
|
||||||
self.truncated = count >= limit as usize;
|
self.truncated = count >= limit as usize;
|
||||||
if emit {
|
if emit {
|
||||||
@@ -129,6 +216,7 @@ struct Subscription {
|
|||||||
next_probe: Instant,
|
next_probe: Instant,
|
||||||
failures: i32,
|
failures: i32,
|
||||||
stable_probes: i32,
|
stable_probes: i32,
|
||||||
|
startup: Option<oneshot::Sender<Result<(), HRESULT>>>,
|
||||||
settings: AlarmFailoverConfig,
|
settings: AlarmFailoverConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -163,6 +251,9 @@ impl AlarmRuntime {
|
|||||||
|
|
||||||
pub fn close(&mut self, proxy: &mut Proxy) {
|
pub fn close(&mut self, proxy: &mut Proxy) {
|
||||||
if let Some(mut subscription) = self.subscription.take() {
|
if let Some(mut subscription) = self.subscription.take() {
|
||||||
|
if let Some(source) = subscription.primary.and_then(|primary| primary.source) {
|
||||||
|
source.finish();
|
||||||
|
}
|
||||||
if let Some(subtags) = &mut subscription.subtags {
|
if let Some(subtags) = &mut subscription.subtags {
|
||||||
subtags.close(proxy);
|
subtags.close(proxy);
|
||||||
}
|
}
|
||||||
@@ -186,8 +277,29 @@ impl AlarmRuntime {
|
|||||||
};
|
};
|
||||||
let probe_interval =
|
let probe_interval =
|
||||||
Duration::from_secs(subscription.settings.failback_probe_interval_seconds as u64);
|
Duration::from_secs(subscription.settings.failback_probe_interval_seconds as u64);
|
||||||
|
let may_request = subscription.mode == Mode::Alarmmgr || now >= subscription.next_probe;
|
||||||
|
let outcome = primary.poll(
|
||||||
|
proxy,
|
||||||
|
self.limit,
|
||||||
|
subscription.mode == Mode::Alarmmgr,
|
||||||
|
may_request,
|
||||||
|
now,
|
||||||
|
);
|
||||||
|
if primary.pending_until.is_some() {
|
||||||
|
self.next_poll = now + self.interval.min(Duration::from_millis(50));
|
||||||
|
}
|
||||||
|
let Some(outcome) = outcome else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if let Some(startup) = subscription.startup.take() {
|
||||||
|
let _ = startup.send(outcome);
|
||||||
|
if outcome.is_err() {
|
||||||
|
self.subscription = None;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
match subscription.mode {
|
match subscription.mode {
|
||||||
Mode::Alarmmgr => match primary.poll(proxy, self.limit, true) {
|
Mode::Alarmmgr => match outcome {
|
||||||
Ok(()) => subscription.failures = 0,
|
Ok(()) => subscription.failures = 0,
|
||||||
Err(hr) if subscription.automatic => {
|
Err(hr) if subscription.automatic => {
|
||||||
subscription.failures += 1;
|
subscription.failures += 1;
|
||||||
@@ -206,9 +318,9 @@ impl AlarmRuntime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
Mode::Subtag if now >= subscription.next_probe => {
|
Mode::Subtag => {
|
||||||
subscription.next_probe = now + probe_interval;
|
subscription.next_probe = now + probe_interval;
|
||||||
if primary.poll(proxy, self.limit, false).is_ok() {
|
if outcome.is_ok() {
|
||||||
subscription.stable_probes += 1;
|
subscription.stable_probes += 1;
|
||||||
if subscription.stable_probes >= subscription.settings.failback_stable_probes {
|
if subscription.stable_probes >= subscription.settings.failback_stable_probes {
|
||||||
subscription.mode = Mode::Alarmmgr;
|
subscription.mode = Mode::Alarmmgr;
|
||||||
@@ -265,8 +377,12 @@ impl AlarmRuntime {
|
|||||||
Mode::Alarmmgr
|
Mode::Alarmmgr
|
||||||
},
|
},
|
||||||
primary: (mode != Mode::Subtag).then(|| Primary {
|
primary: (mode != Mode::Subtag).then(|| Primary {
|
||||||
|
local_manager: automatic
|
||||||
|
.then(|| LocalAlarmmgr::for_subscription(&request.subscription_expression))
|
||||||
|
.flatten(),
|
||||||
expression: request.subscription_expression,
|
expression: request.subscription_expression,
|
||||||
consumer: None,
|
source: None,
|
||||||
|
pending_until: None,
|
||||||
records: BTreeMap::new(),
|
records: BTreeMap::new(),
|
||||||
truncated: false,
|
truncated: false,
|
||||||
}),
|
}),
|
||||||
@@ -275,47 +391,43 @@ impl AlarmRuntime {
|
|||||||
next_probe: Instant::now(),
|
next_probe: Instant::now(),
|
||||||
failures: 0,
|
failures: 0,
|
||||||
stable_probes: 0,
|
stable_probes: 0,
|
||||||
|
startup: None,
|
||||||
settings,
|
settings,
|
||||||
};
|
};
|
||||||
let mut initial_failure = None;
|
|
||||||
if let Some(primary) = &mut subscription.primary {
|
|
||||||
match WnWrap::subscribe(&primary.expression) {
|
|
||||||
Ok(consumer) => primary.consumer = Some(consumer),
|
|
||||||
Err(hr) if automatic => {
|
|
||||||
subscription.failures = 1;
|
|
||||||
initial_failure = Some(hr);
|
|
||||||
}
|
|
||||||
Err(hr) => return Err(AlarmError::failed("WnWrap subscribe", hr)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if automatic || mode == Mode::Subtag {
|
if automatic || mode == Mode::Subtag {
|
||||||
subscription.subtags = Some(Subtags::attach(proxy, request.watch_list)?);
|
subscription.subtags = Some(Subtags::attach(proxy, request.watch_list)?);
|
||||||
}
|
}
|
||||||
if let Some(hr) =
|
|
||||||
initial_failure.filter(|_| subscription.settings.consecutive_failure_threshold == 1)
|
|
||||||
{
|
|
||||||
subscription.mode = Mode::Subtag;
|
|
||||||
subscription.next_probe = Instant::now()
|
|
||||||
+ Duration::from_secs(subscription.settings.failback_probe_interval_seconds as u64);
|
|
||||||
publish_mode(
|
|
||||||
&proxy.publisher,
|
|
||||||
Mode::Subtag,
|
|
||||||
"primary subscribe failed",
|
|
||||||
hr,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
self.subscription = Some(subscription);
|
self.subscription = Some(subscription);
|
||||||
self.next_poll = Instant::now();
|
self.next_poll = Instant::now();
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn execute(&mut self, proxy: &mut Proxy, command: MxCommand) -> MxCommandReply {
|
pub fn execute(
|
||||||
|
&mut self,
|
||||||
|
proxy: &mut Proxy,
|
||||||
|
command: MxCommand,
|
||||||
|
) -> (
|
||||||
|
MxCommandReply,
|
||||||
|
Option<oneshot::Receiver<Result<(), HRESULT>>>,
|
||||||
|
) {
|
||||||
let mut reply = MxCommandReply {
|
let mut reply = MxCommandReply {
|
||||||
kind: command.kind,
|
kind: command.kind,
|
||||||
hresult: Some(0),
|
hresult: Some(0),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
let result = self.dispatch(proxy, command, &mut reply);
|
let result = self.dispatch(proxy, command, &mut reply);
|
||||||
|
let mut readiness = None;
|
||||||
|
if result.is_ok() && reply.kind() == Kind::SubscribeAlarms {
|
||||||
|
if let Some(subscription) = self
|
||||||
|
.subscription
|
||||||
|
.as_mut()
|
||||||
|
.filter(|subscription| subscription.primary.is_some() && !subscription.automatic)
|
||||||
|
{
|
||||||
|
let (sender, receiver) = oneshot::channel();
|
||||||
|
subscription.startup = Some(sender);
|
||||||
|
readiness = Some(receiver);
|
||||||
|
}
|
||||||
|
}
|
||||||
let (code, message) = match result {
|
let (code, message) = match result {
|
||||||
Ok(()) => (Code::Ok, std::mem::take(&mut reply.diagnostic_message)),
|
Ok(()) => (Code::Ok, std::mem::take(&mut reply.diagnostic_message)),
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
@@ -336,7 +448,7 @@ impl AlarmRuntime {
|
|||||||
message: message.clone(),
|
message: message.clone(),
|
||||||
});
|
});
|
||||||
reply.diagnostic_message = message;
|
reply.diagnostic_message = message;
|
||||||
reply
|
(reply, readiness)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn dispatch(
|
fn dispatch(
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
use std::mem::size_of;
|
||||||
|
use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle};
|
||||||
|
|
||||||
|
use windows_sys::Win32::Foundation::{INVALID_HANDLE_VALUE, WAIT_OBJECT_0};
|
||||||
|
use windows_sys::Win32::System::Diagnostics::ToolHelp::{
|
||||||
|
CreateToolhelp32Snapshot, Process32FirstW, Process32NextW, PROCESSENTRY32W, TH32CS_SNAPPROCESS,
|
||||||
|
};
|
||||||
|
use windows_sys::Win32::System::RemoteDesktop::ProcessIdToSessionId;
|
||||||
|
use windows_sys::Win32::System::Threading::{
|
||||||
|
OpenProcess, WaitForSingleObject, PROCESS_SYNCHRONIZE,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub enum LocalAlarmmgr {
|
||||||
|
Unobserved,
|
||||||
|
Running(OwnedHandle),
|
||||||
|
Stopped,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LocalAlarmmgr {
|
||||||
|
pub fn for_subscription(expression: &str) -> Option<Self> {
|
||||||
|
let (node, query) = expression.trim_start_matches('\\').split_once('\\')?;
|
||||||
|
if !query.split('!').next()?.eq_ignore_ascii_case("Galaxy") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let computer = std::env::var("COMPUTERNAME").unwrap_or_default();
|
||||||
|
(node == "."
|
||||||
|
|| node.eq_ignore_ascii_case("localhost")
|
||||||
|
|| (!computer.is_empty() && node.eq_ignore_ascii_case(&computer)))
|
||||||
|
.then_some(Self::Unobserved)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn available(&mut self) -> bool {
|
||||||
|
if let Self::Running(process) = self {
|
||||||
|
// SAFETY: this is an owned, waitable process handle. A zero timeout never blocks.
|
||||||
|
if unsafe { WaitForSingleObject(process.as_raw_handle(), 0) } != WAIT_OBJECT_0 {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
*self = Self::Stopped;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if let Some(process) = find_alarmmgr() {
|
||||||
|
*self = Self::Running(process);
|
||||||
|
}
|
||||||
|
// A process we could never observe does not override COM health.
|
||||||
|
!matches!(self, Self::Stopped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn find_alarmmgr() -> Option<OwnedHandle> {
|
||||||
|
// SAFETY: a process snapshot owns its handle independently of the processes it enumerates.
|
||||||
|
let snapshot = unsafe { CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) };
|
||||||
|
if snapshot == INVALID_HANDLE_VALUE {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let snapshot = unsafe { OwnedHandle::from_raw_handle(snapshot) };
|
||||||
|
let mut entry = PROCESSENTRY32W {
|
||||||
|
dwSize: size_of::<PROCESSENTRY32W>() as u32,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
// SAFETY: the entry is initialized with the size expected by both enumeration functions.
|
||||||
|
let mut present = unsafe { Process32FirstW(snapshot.as_raw_handle(), &mut entry) };
|
||||||
|
while present != 0 {
|
||||||
|
let name = String::from_utf16_lossy(
|
||||||
|
&entry.szExeFile[..entry
|
||||||
|
.szExeFile
|
||||||
|
.iter()
|
||||||
|
.position(|&c| c == 0)
|
||||||
|
.unwrap_or(entry.szExeFile.len())],
|
||||||
|
);
|
||||||
|
let mut session = u32::MAX;
|
||||||
|
// The Galaxy provider uses Session 0. Interactive InTouch instances are independent.
|
||||||
|
if name.eq_ignore_ascii_case("alarmmgr.exe")
|
||||||
|
&& unsafe { ProcessIdToSessionId(entry.th32ProcessID, &mut session) } != 0
|
||||||
|
&& session == 0
|
||||||
|
{
|
||||||
|
// SAFETY: Windows either returns a new owned handle with wait access or returns null.
|
||||||
|
let process = unsafe { OpenProcess(PROCESS_SYNCHRONIZE, 0, entry.th32ProcessID) };
|
||||||
|
if !process.is_null() {
|
||||||
|
return Some(unsafe { OwnedHandle::from_raw_handle(process) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
present = unsafe { Process32NextW(snapshot.as_raw_handle(), &mut entry) };
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
@@ -131,17 +131,14 @@ pub fn full_reference(provider: &str, group: &str, name: &str) -> String {
|
|||||||
format!("{provider}{separator}{group}{dot}{name}")
|
format!("{provider}{separator}{group}{dot}{name}")
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse_snapshot(xml: &str) -> Result<(BTreeMap<Uuid, Record>, usize), roxmltree::Error> {
|
pub fn parse_snapshot(xml: &str) -> Option<(BTreeMap<Uuid, Record>, usize)> {
|
||||||
let mut records = BTreeMap::new();
|
let document = roxmltree::Document::parse(xml).ok()?;
|
||||||
if xml.trim().is_empty() {
|
|
||||||
return Ok((records, 0));
|
|
||||||
}
|
|
||||||
let document = roxmltree::Document::parse(xml)?;
|
|
||||||
let root = document.root_element();
|
let root = document.root_element();
|
||||||
if !unqualified(root, "ALARM_RECORDS") {
|
if !unqualified(root, "ALARM_RECORDS") {
|
||||||
return Ok((records, 0));
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut records = BTreeMap::new();
|
||||||
let mut count = 0;
|
let mut count = 0;
|
||||||
for alarm in root.children().filter(|&child| unqualified(child, "ALARM")) {
|
for alarm in root.children().filter(|&child| unqualified(child, "ALARM")) {
|
||||||
count += 1;
|
count += 1;
|
||||||
@@ -182,7 +179,7 @@ pub fn parse_snapshot(xml: &str) -> Result<(BTreeMap<Uuid, Record>, usize), roxm
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Ok((records, count))
|
Some((records, count))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn unqualified(node: roxmltree::Node<'_, '_>, name: &str) -> bool {
|
fn unqualified(node: roxmltree::Node<'_, '_>, name: &str) -> bool {
|
||||||
|
|||||||
@@ -0,0 +1,192 @@
|
|||||||
|
//! The helper uses the same executable. Only it loads WnWrap, so a stuck
|
||||||
|
//! registration, poll, or teardown can be terminated without stopping MXAccess.
|
||||||
|
|
||||||
|
use std::io::{self, Read, Write};
|
||||||
|
use std::mem::size_of;
|
||||||
|
use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle};
|
||||||
|
use std::os::windows::process::CommandExt;
|
||||||
|
use std::process::{Child, ChildStdin, Command, ExitCode, Stdio};
|
||||||
|
use std::ptr::null;
|
||||||
|
use std::sync::mpsc::{self, Receiver};
|
||||||
|
use std::thread;
|
||||||
|
|
||||||
|
use windows_sys::core::HRESULT;
|
||||||
|
use windows_sys::Win32::System::JobObjects::{
|
||||||
|
AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
|
||||||
|
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
|
||||||
|
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
|
||||||
|
};
|
||||||
|
use windows_sys::Win32::System::Threading::{WaitForSingleObject, CREATE_NO_WINDOW};
|
||||||
|
|
||||||
|
use super::wnwrap::WnWrap;
|
||||||
|
use crate::mxaccess::{pump_messages, Apartment};
|
||||||
|
|
||||||
|
pub const ARGUMENT: &str = "--wnwrap-snapshot-worker";
|
||||||
|
const MAX_SNAPSHOT_BYTES: usize = 128 * 1024 * 1024;
|
||||||
|
|
||||||
|
pub struct SnapshotProcess {
|
||||||
|
// Closing the job terminates the child even during an uninterruptible COM call.
|
||||||
|
job: OwnedHandle,
|
||||||
|
child: Child,
|
||||||
|
requests: ChildStdin,
|
||||||
|
pub replies: Receiver<Result<String, HRESULT>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SnapshotProcess {
|
||||||
|
pub fn start(expression: &str, limit: i32) -> io::Result<Self> {
|
||||||
|
let job = kill_on_close_job()?;
|
||||||
|
let mut child = Command::new(std::env::current_exe()?)
|
||||||
|
.args([ARGUMENT, expression, &limit.to_string()])
|
||||||
|
.env_remove("MXGATEWAY_WORKER_NONCE")
|
||||||
|
.stdin(Stdio::piped())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.creation_flags(CREATE_NO_WINDOW)
|
||||||
|
.spawn()?;
|
||||||
|
// SAFETY: both handles remain owned in this scope.
|
||||||
|
if unsafe { AssignProcessToJobObject(job.as_raw_handle(), child.as_raw_handle()) } == 0 {
|
||||||
|
let error = io::Error::last_os_error();
|
||||||
|
let _ = child.kill();
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
let requests = child.stdin.take().expect("piped stdin");
|
||||||
|
let mut output = child.stdout.take().expect("piped stdout");
|
||||||
|
let (sender, replies) = mpsc::sync_channel(1);
|
||||||
|
thread::Builder::new()
|
||||||
|
.name("wnwrap-snapshot-reader".into())
|
||||||
|
.spawn(move || {
|
||||||
|
while let Ok(reply) = read_reply(&mut output) {
|
||||||
|
if sender.send(reply).is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
Ok(Self {
|
||||||
|
job,
|
||||||
|
child,
|
||||||
|
requests,
|
||||||
|
replies,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn finish(self) {
|
||||||
|
let Self {
|
||||||
|
job,
|
||||||
|
child,
|
||||||
|
requests,
|
||||||
|
replies: _,
|
||||||
|
} = self;
|
||||||
|
drop(requests);
|
||||||
|
// Give an idle helper time to deregister. A COM teardown that stalls
|
||||||
|
// cannot delay UnsubscribeAlarms or keep the job alive indefinitely.
|
||||||
|
let _ = thread::Builder::new()
|
||||||
|
.name("wnwrap-snapshot-cleanup".into())
|
||||||
|
.spawn(move || {
|
||||||
|
// SAFETY: the child process handle is owned by this thread.
|
||||||
|
unsafe { WaitForSingleObject(child.as_raw_handle(), 250) };
|
||||||
|
drop(job);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn request(&mut self) -> io::Result<()> {
|
||||||
|
// The parent allows only one outstanding byte on the request pipe.
|
||||||
|
self.requests.write_all(&[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn kill_on_close_job() -> io::Result<OwnedHandle> {
|
||||||
|
// SAFETY: null arguments create an unnamed job with default security.
|
||||||
|
let raw = unsafe { CreateJobObjectW(null(), null()) };
|
||||||
|
if raw.is_null() {
|
||||||
|
return Err(io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
let job = unsafe { OwnedHandle::from_raw_handle(raw) };
|
||||||
|
let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
|
||||||
|
limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
|
||||||
|
// SAFETY: the settings structure matches the job information class.
|
||||||
|
if unsafe {
|
||||||
|
SetInformationJobObject(
|
||||||
|
job.as_raw_handle(),
|
||||||
|
JobObjectExtendedLimitInformation,
|
||||||
|
(&limits as *const JOBOBJECT_EXTENDED_LIMIT_INFORMATION).cast(),
|
||||||
|
size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,
|
||||||
|
)
|
||||||
|
} == 0
|
||||||
|
{
|
||||||
|
return Err(io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
Ok(job)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run() -> ExitCode {
|
||||||
|
let mut args = std::env::args().skip(2);
|
||||||
|
let Some(expression) = args.next() else {
|
||||||
|
return ExitCode::FAILURE;
|
||||||
|
};
|
||||||
|
let Some(limit) = args.next().and_then(|value| value.parse().ok()) else {
|
||||||
|
return ExitCode::FAILURE;
|
||||||
|
};
|
||||||
|
let Ok(_apartment) = Apartment::initialize() else {
|
||||||
|
return ExitCode::FAILURE;
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut input = io::stdin().lock();
|
||||||
|
let mut output = io::stdout().lock();
|
||||||
|
let mut subscription = None;
|
||||||
|
let mut request = [0];
|
||||||
|
while input.read_exact(&mut request).is_ok() && request == [1] && pump_messages() {
|
||||||
|
let reply = match &subscription {
|
||||||
|
Some(consumer) => WnWrap::snapshot(consumer, limit),
|
||||||
|
None => WnWrap::subscribe(&expression).and_then(|consumer| {
|
||||||
|
let result = consumer.snapshot(limit);
|
||||||
|
subscription = Some(consumer);
|
||||||
|
result
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
let failed = reply.is_err();
|
||||||
|
if write_reply(&mut output, reply).is_err() || failed {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ExitCode::SUCCESS
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_reply(mut output: impl Write, reply: Result<String, HRESULT>) -> io::Result<()> {
|
||||||
|
match reply {
|
||||||
|
Err(hr) => output.write_all(&hr.to_le_bytes())?,
|
||||||
|
Ok(xml) if xml.len() <= MAX_SNAPSHOT_BYTES => {
|
||||||
|
output.write_all(&0_i32.to_le_bytes())?;
|
||||||
|
output.write_all(&(xml.len() as u32).to_le_bytes())?;
|
||||||
|
output.write_all(xml.as_bytes())?;
|
||||||
|
}
|
||||||
|
Ok(_) => {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"WnWrap snapshot is too large",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
output.flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_reply(mut input: impl Read) -> io::Result<Result<String, HRESULT>> {
|
||||||
|
let mut field = [0; 4];
|
||||||
|
input.read_exact(&mut field)?;
|
||||||
|
let hr = i32::from_le_bytes(field);
|
||||||
|
if hr != 0 {
|
||||||
|
return Ok(Err(hr));
|
||||||
|
}
|
||||||
|
input.read_exact(&mut field)?;
|
||||||
|
let length = u32::from_le_bytes(field) as usize;
|
||||||
|
if length > MAX_SNAPSHOT_BYTES {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"WnWrap snapshot is too large",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut bytes = vec![0; length];
|
||||||
|
input.read_exact(&mut bytes)?;
|
||||||
|
String::from_utf8(bytes)
|
||||||
|
.map(Ok)
|
||||||
|
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))
|
||||||
|
}
|
||||||
@@ -122,15 +122,16 @@ impl WnWrap {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn snapshot(&self, limit: i32) -> Result<String, HRESULT> {
|
pub fn snapshot(&self, limit: i32) -> Result<String, HRESULT> {
|
||||||
// The live test can fail this worker's polls without stopping the shared Alarmmgr.
|
// The debug-only seam can stall this helper without stopping shared AVEVA services.
|
||||||
#[cfg(all(debug_assertions, feature = "alarm-test-faults"))]
|
#[cfg(all(debug_assertions, feature = "alarm-test-faults"))]
|
||||||
if self
|
if let Some(marker) = &self.snapshot_fault_marker {
|
||||||
.snapshot_fault_marker
|
if let Ok(fault) = std::fs::read(marker) {
|
||||||
.as_ref()
|
if fault == b"stall" {
|
||||||
.is_some_and(|path| path.is_file())
|
std::thread::sleep(std::time::Duration::from_secs(60));
|
||||||
{
|
}
|
||||||
return Err(E_FAIL);
|
return Err(E_FAIL);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
struct XmlVariant(VARIANT);
|
struct XmlVariant(VARIANT);
|
||||||
impl Drop for XmlVariant {
|
impl Drop for XmlVariant {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
|
|||||||
@@ -17,6 +17,11 @@ mod wire;
|
|||||||
#[cfg(all(windows, target_arch = "x86"))]
|
#[cfg(all(windows, target_arch = "x86"))]
|
||||||
#[tokio::main(flavor = "current_thread")]
|
#[tokio::main(flavor = "current_thread")]
|
||||||
async fn main() -> std::process::ExitCode {
|
async fn main() -> std::process::ExitCode {
|
||||||
|
if std::env::args_os().nth(1).as_deref()
|
||||||
|
== Some(std::ffi::OsStr::new(alarm::snapshot_worker::ARGUMENT))
|
||||||
|
{
|
||||||
|
return alarm::snapshot_worker::run();
|
||||||
|
}
|
||||||
match session::run().await {
|
match session::run().await {
|
||||||
Ok(()) => std::process::ExitCode::SUCCESS,
|
Ok(()) => std::process::ExitCode::SUCCESS,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
|
|||||||
@@ -118,7 +118,10 @@ impl From<WriteValueError> for CommandError {
|
|||||||
|
|
||||||
pub enum CommandResult {
|
pub enum CommandResult {
|
||||||
Done,
|
Done,
|
||||||
Alarm(Box<MxCommandReply>),
|
Alarm(
|
||||||
|
Box<MxCommandReply>,
|
||||||
|
Option<oneshot::Receiver<Result<(), HRESULT>>>,
|
||||||
|
),
|
||||||
Handle(i32),
|
Handle(i32),
|
||||||
Status(MxStatusProxy),
|
Status(MxStatusProxy),
|
||||||
SubscribeBulk(Vec<SubscribeResult>),
|
SubscribeBulk(Vec<SubscribeResult>),
|
||||||
@@ -283,9 +286,10 @@ fn pump_loop(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let result = match request.command {
|
let result = match request.command {
|
||||||
Command::Alarm(command) => Ok(CommandResult::Alarm(Box::new(
|
Command::Alarm(command) => {
|
||||||
alarms.execute(proxy, *command),
|
let (reply, startup) = alarms.execute(proxy, *command);
|
||||||
))),
|
Ok(CommandResult::Alarm(Box::new(reply), startup))
|
||||||
|
}
|
||||||
command => proxy.execute(command),
|
command => proxy.execute(command),
|
||||||
};
|
};
|
||||||
let _ = request.reply.send(result);
|
let _ = request.reply.send(result);
|
||||||
@@ -317,10 +321,10 @@ pub(crate) fn pump_messages() -> bool {
|
|||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Apartment;
|
pub(crate) struct Apartment;
|
||||||
|
|
||||||
impl Apartment {
|
impl Apartment {
|
||||||
fn initialize() -> Result<Self, HRESULT> {
|
pub(crate) fn initialize() -> Result<Self, HRESULT> {
|
||||||
// SAFETY: this runs on the new dedicated thread and is balanced by Drop.
|
// SAFETY: this runs on the new dedicated thread and is balanced by Drop.
|
||||||
check(unsafe { CoInitializeEx(null(), COINIT_APARTMENTTHREADED as u32) })?;
|
check(unsafe { CoInitializeEx(null(), COINIT_APARTMENTTHREADED as u32) })?;
|
||||||
Ok(Self)
|
Ok(Self)
|
||||||
|
|||||||
@@ -652,13 +652,26 @@ async fn execute(
|
|||||||
};
|
};
|
||||||
let kind = command.kind();
|
let kind = command.kind();
|
||||||
if alarm::handles(kind) {
|
if alarm::handles(kind) {
|
||||||
let Ok(CommandResult::Alarm(mut result)) = sta
|
let Ok(CommandResult::Alarm(mut result, startup)) = sta
|
||||||
.execute(Command::Alarm(Box::new(command)))
|
.execute(Command::Alarm(Box::new(command)))
|
||||||
.await
|
.await
|
||||||
.map_err(Error::sta)?
|
.map_err(Error::sta)?
|
||||||
else {
|
else {
|
||||||
unreachable!("STA alarm route returns an alarm reply");
|
unreachable!("STA alarm route returns an alarm reply");
|
||||||
};
|
};
|
||||||
|
if let Some(startup) = startup {
|
||||||
|
if let Err(hr) = startup
|
||||||
|
.await
|
||||||
|
.unwrap_or(Err(windows_sys::Win32::Foundation::E_FAIL))
|
||||||
|
{
|
||||||
|
result.hresult = Some(hr);
|
||||||
|
set_status(
|
||||||
|
&mut result,
|
||||||
|
ProtocolStatusCode::MxaccessFailure,
|
||||||
|
&format!("WnWrap subscribe failed (HRESULT 0x{:08X})", hr as u32),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
result.session_id = reply.session_id;
|
result.session_id = reply.session_id;
|
||||||
result.correlation_id = reply.correlation_id;
|
result.correlation_id = reply.correlation_id;
|
||||||
return Ok(*result);
|
return Ok(*result);
|
||||||
@@ -921,7 +934,7 @@ async fn execute(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
match sta.execute(native).await.map_err(Error::sta)? {
|
match sta.execute(native).await.map_err(Error::sta)? {
|
||||||
Ok(CommandResult::Alarm(_)) => unreachable!("data command returned an alarm reply"),
|
Ok(CommandResult::Alarm(..)) => unreachable!("data command returned an alarm reply"),
|
||||||
Ok(CommandResult::Handle(handle)) => {
|
Ok(CommandResult::Handle(handle)) => {
|
||||||
if !matches!(kind, Kind::AuthenticateUser | Kind::ArchestraUserToId) {
|
if !matches!(kind, Kind::AuthenticateUser | Kind::ArchestraUserToId) {
|
||||||
reply.return_value = Some(int32(handle));
|
reply.return_value = Some(int32(handle));
|
||||||
|
|||||||
Reference in New Issue
Block a user