fix(worker): review 2026-10-21 stream E — cache eviction, phased ReadBulk, poll/STA fault handling
ci / nightly-windev (push) Has been skipped
ci / windows-x86 (push) Failing after 1m3s
ci / java (push) Successful in 2m17s
ci / portable (push) Failing after 4m42s

- RemoveItem/Unregister evict value + write-completion caches (leak under churn)
- ReadBulk snapshot path: add/advise all, one shared deadline, then tear down
- alarm poll faults only after 3 consecutive failures; transient ones logged
- AlarmCommandHandler disposes primary when standby build throws
- StaRuntime pumps between queued work items
- WRK-02: post-start STA loop death surfaces as StaHung WorkerFault
- WRK-09: AppDomain unhandled-exception hook, exit code 7
- RETRIGGER documented as reserved, never synthesized
- TST-05: live facts for Ping/GetSessionState/GetWorkerInfo/DrainEvents/ShutdownWorker
This commit is contained in:
Joseph Doherty
2026-10-22 06:19:32 -04:00
parent 488649ce99
commit 7618cafb6c
22 changed files with 1859 additions and 90 deletions
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using ZB.MOM.WW.MxGateway.Contracts;
@@ -121,6 +122,49 @@ public sealed class WorkerApplicationTests
Assert.Equal(typeof(InvalidOperationException).FullName, entry.Fields["exception_type"]);
}
/// <summary>
/// WRK-09: the AppDomain unhandled-exception hook logs the exception type
/// (never its message) and exits with the dedicated exit code.
/// </summary>
[Fact]
public void HandleUnhandledException_LogsTypeOnlyAndExitsWithUnhandledExceptionCode()
{
MemoryWorkerLogger logger = new();
int? exitCode = null;
ZB.MOM.WW.MxGateway.Worker.WorkerApplication.HandleUnhandledException(
logger,
new InvalidOperationException("password=hunter2"),
isTerminating: true,
code => exitCode = code);
Assert.Equal((int)WorkerExitCode.UnhandledException, exitCode);
Assert.Equal(7, (int)WorkerExitCode.UnhandledException);
MemoryWorkerLogEntry entry = Assert.Single(logger.Entries);
Assert.Equal("Error", entry.Level);
Assert.Equal("WorkerUnhandledException", entry.EventName);
Assert.Equal(typeof(InvalidOperationException).FullName, entry.Fields["exception_type"]);
Assert.True(entry.Fields["is_terminating"] is true);
Assert.DoesNotContain(
entry.Fields.Values,
value => value?.ToString()?.Contains("hunter2") == true);
}
/// <summary>A failing logger must not prevent the defined exit.</summary>
[Fact]
public void HandleUnhandledException_WhenLoggerThrows_StillExits()
{
int? exitCode = null;
ZB.MOM.WW.MxGateway.Worker.WorkerApplication.HandleUnhandledException(
new ThrowingLogger(),
exceptionObject: null,
isTerminating: false,
code => exitCode = code);
Assert.Equal((int)WorkerExitCode.UnhandledException, exitCode);
}
private static string[] ValidArgs(string? protocolVersion = null)
{
return
@@ -171,4 +215,15 @@ public sealed class WorkerApplicationTests
throw _exception;
}
}
private sealed class ThrowingLogger : IWorkerLogger
{
/// <inheritdoc />
public void Information(string eventName, IReadOnlyDictionary<string, object?> fields) =>
throw new InvalidOperationException("logger failed");
/// <inheritdoc />
public void Error(string eventName, IReadOnlyDictionary<string, object?> fields) =>
throw new InvalidOperationException("logger failed");
}
}
@@ -499,6 +499,40 @@ public sealed class AlarmCommandHandlerTests
Assert.Equal(AlarmProviderMode.Subtag, evt.OnAlarmProviderModeChanged.Mode);
}
/// <summary>
/// In auto mode the primary consumer is built before the standby. When
/// building the standby throws, the primary is owned by nothing — the
/// failed Subscribe never reaches the dispatcher that would dispose it —
/// so the handler must dispose it rather than leak its COM object, and
/// the original failure must still propagate.
/// </summary>
[Fact]
public void Subscribe_AutoModeWhenStandbyBuildThrows_DisposesPrimaryAndRethrows()
{
FakeConsumer primary = new FakeConsumer();
InvalidOperationException standbyFailure = new InvalidOperationException("standby build failed");
AlarmCommandHandler handler = new AlarmCommandHandler(
new MxAccessEventQueue(),
() => primary,
threadAffinityCheck: null,
comFactory: null,
standbyFactory: _ => throw standbyFailure);
SubscribeAlarmsCommand command = new SubscribeAlarmsCommand
{
SubscriptionExpression = @"\\HOST\Galaxy!Area",
ForcedMode = AlarmProviderMode.Unspecified,
};
command.WatchList.Add(NewTarget("Galaxy!Area.Tank01.Level.HiHi"));
InvalidOperationException thrown = Assert.Throws<InvalidOperationException>(
() => handler.Subscribe(command, "s1"));
Assert.Same(standbyFailure, thrown);
Assert.True(primary.Disposed, "The primary consumer built before the failing standby must be disposed.");
Assert.False(handler.IsSubscribed);
}
/// <summary>An explicitly fallback-capable auto subscription builds a replaceable standby even when discovery is initially empty.</summary>
[Fact]
public void Subscribe_AutoModeWithEmptyWatchListAndFallbackEnabled_BuildsReplaceableStandby()
@@ -102,6 +102,22 @@ public sealed class AlarmRecordTransitionMapperTests
Assert.Equal(expected, AlarmRecordTransitionMapper.MapTransition(previous, current));
}
/// <summary>
/// RETRIGGER is reserved and never synthesized: no state pair — including
/// cleared-to-active, the re-alarm shape — maps to it.
/// </summary>
[Fact]
public void MapTransition_ForEveryStatePair_NeverProducesRetrigger()
{
foreach (MxAlarmStateKind previous in System.Enum.GetValues(typeof(MxAlarmStateKind)))
{
foreach (MxAlarmStateKind current in System.Enum.GetValues(typeof(MxAlarmStateKind)))
{
Assert.NotEqual(AlarmTransitionKind.Retrigger, AlarmRecordTransitionMapper.MapTransition(previous, current));
}
}
}
/// <summary>Verifies that valid XML fields assemble into correct UTC timestamp.</summary>
[Fact]
public void ParseTransitionTimestampUtc_WithValidXmlFields_AssemblesUtc()
@@ -0,0 +1,448 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Google.Protobuf.Collections;
using Google.Protobuf.WellKnownTypes;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Worker.MxAccess;
using ZB.MOM.WW.MxGateway.Worker.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Worker.Tests.MxAccess;
/// <summary>
/// Direct tests of <see cref="MxAccessSession"/> handle-lifetime cache
/// eviction and the phased ReadBulk snapshot path, driven through
/// <see cref="MxAccessSession.CreateForTesting"/> with a recording fake
/// server. Nothing here needs an STA: the session methods are plain calls,
/// and ReadBulk's pump step is a test delegate that stands in for the COM
/// OnDataChange dispatch by writing the value cache directly.
/// </summary>
public sealed class MxAccessSessionTests
{
private const string ReadBulkTimeoutText = "timed out";
/// <summary>
/// RemoveItem must evict the handle pair's OnWriteComplete completion
/// entry as well as its last-value entry; otherwise an
/// AddItem -> Write -> RemoveItem churn grows the completion cache by one
/// entry per cycle for the life of the worker.
/// </summary>
[Fact]
public void RemoveItem_EvictsValueAndWriteCompletionEntriesForTheHandle()
{
RecordingMxAccessServer server = new();
CompletionCacheSink sink = new();
MxAccessValueCache valueCache = new();
using MxAccessSession session = MxAccessSession.CreateForTesting(server, sink, valueCache: valueCache);
int serverHandle = session.Register("client");
int keep = session.AddItem(serverHandle, "Tag.Keep");
int drop = session.AddItem(serverHandle, "Tag.Drop");
RecordCompletion(sink.WriteCompletionCache, serverHandle, keep);
RecordCompletion(sink.WriteCompletionCache, serverHandle, drop);
valueCache.Set(serverHandle, keep, BuildDataChange(serverHandle, keep, 1));
valueCache.Set(serverHandle, drop, BuildDataChange(serverHandle, drop, 2));
session.RemoveItem(serverHandle, drop);
Assert.Equal(1, sink.WriteCompletionCache.Count);
Assert.Equal(0UL, sink.WriteCompletionCache.CurrentVersion(serverHandle, drop));
Assert.Equal(1UL, sink.WriteCompletionCache.CurrentVersion(serverHandle, keep));
Assert.False(valueCache.TryGet(serverHandle, drop, out _));
Assert.True(valueCache.TryGet(serverHandle, keep, out _));
}
/// <summary>
/// A long-lived writer that keeps adding, writing, and removing items
/// must leave the completion cache empty — the leak this fix closes.
/// </summary>
[Fact]
public void AddItemWriteRemoveItemChurn_DoesNotGrowWriteCompletionCache()
{
RecordingMxAccessServer server = new();
CompletionCacheSink sink = new();
using MxAccessSession session = MxAccessSession.CreateForTesting(server, sink);
int serverHandle = session.Register("client");
for (int cycle = 0; cycle < 50; cycle++)
{
int itemHandle = session.AddItem(serverHandle, "Tag.Churn");
session.Write(serverHandle, itemHandle, cycle, userId: 0);
// Stand in for the sink's OnWriteComplete post-publish hook.
RecordCompletion(sink.WriteCompletionCache, serverHandle, itemHandle);
session.RemoveItem(serverHandle, itemHandle);
}
Assert.Equal(0, sink.WriteCompletionCache.Count);
}
/// <summary>
/// Unregister drops every item under the server in one MXAccess call,
/// so the per-item eviction never runs for them: both caches must be
/// evicted by server handle, and other servers' entries kept.
/// </summary>
[Fact]
public void Unregister_EvictsValueAndWriteCompletionEntriesForThatServerOnly()
{
RecordingMxAccessServer server = new();
CompletionCacheSink sink = new();
MxAccessValueCache valueCache = new();
using MxAccessSession session = MxAccessSession.CreateForTesting(server, sink, valueCache: valueCache);
int doomed = session.Register("doomed");
int survivor = session.Register("survivor");
int doomedItemA = session.AddItem(doomed, "Tag.A");
int doomedItemB = session.AddItem(doomed, "Tag.B");
int survivorItem = session.AddItem(survivor, "Tag.C");
RecordCompletion(sink.WriteCompletionCache, doomed, doomedItemA);
RecordCompletion(sink.WriteCompletionCache, doomed, doomedItemB);
RecordCompletion(sink.WriteCompletionCache, survivor, survivorItem);
valueCache.Set(doomed, doomedItemA, BuildDataChange(doomed, doomedItemA, 1));
valueCache.Set(doomed, doomedItemB, BuildDataChange(doomed, doomedItemB, 2));
valueCache.Set(survivor, survivorItem, BuildDataChange(survivor, survivorItem, 3));
session.Unregister(doomed);
Assert.Equal(1, sink.WriteCompletionCache.Count);
Assert.Equal(1UL, sink.WriteCompletionCache.CurrentVersion(survivor, survivorItem));
Assert.Equal(1, valueCache.Count);
Assert.True(valueCache.TryGet(survivor, survivorItem, out _));
Assert.False(valueCache.TryGet(doomed, doomedItemA, out _));
Assert.False(valueCache.TryGet(doomed, doomedItemB, out _));
}
/// <summary>
/// N uncached tags that never deliver an OnDataChange must cost one
/// shared timeout, not N x timeout, while each still reports its own
/// timed-out failure with its own item handle.
/// </summary>
[Fact]
public void ReadBulk_UncachedTagsThatNeverDeliver_WaitOneSharedTimeoutAndEachTimesOut()
{
RecordingMxAccessServer server = new();
using MxAccessSession session = MxAccessSession.CreateForTesting(server, new NoopEventSink());
int serverHandle = session.Register("client");
string[] tags = { "Tag.1", "Tag.2", "Tag.3", "Tag.4", "Tag.5" };
TimeSpan timeout = TimeSpan.FromMilliseconds(300);
Stopwatch stopwatch = Stopwatch.StartNew();
IReadOnlyList<BulkReadResult> results = session.ReadBulk(serverHandle, tags, timeout, () => { });
stopwatch.Stop();
// Serial would be >= 5 x 300 ms = 1500 ms. Allow generous slack for a
// loaded CI host while still failing the serial shape.
Assert.True(
stopwatch.Elapsed < TimeSpan.FromMilliseconds(1200),
$"ReadBulk took {stopwatch.Elapsed.TotalMilliseconds:F0} ms; expected one shared {timeout.TotalMilliseconds:F0} ms wait.");
Assert.True(stopwatch.Elapsed >= TimeSpan.FromMilliseconds(250));
Assert.Equal(tags.Length, results.Count);
HashSet<int> handles = new();
for (int index = 0; index < tags.Length; index++)
{
BulkReadResult result = results[index];
Assert.Equal(tags[index], result.TagAddress);
Assert.False(result.WasSuccessful);
Assert.False(result.WasCached);
Assert.Equal(serverHandle, result.ServerHandle);
Assert.Equal(server.HandleForTag(tags[index]), result.ItemHandle);
Assert.Equal(
$"ReadBulk timed out after {timeout.TotalMilliseconds:F0} ms waiting for first OnDataChange.",
result.ErrorMessage);
Assert.True(handles.Add(result.ItemHandle));
}
// Every snapshot handle is still torn down.
foreach (int handle in handles)
{
Assert.Contains($"UnAdvise:{serverHandle}:{handle}", server.Operations);
Assert.Contains($"RemoveItem:{serverHandle}:{handle}", server.Operations);
}
Assert.Empty(session.HandleRegistry.ItemHandles);
}
/// <summary>
/// The phased lifecycle: every uncached tag is added and advised before
/// the wait, and none is torn down until the wait is over.
/// </summary>
[Fact]
public void ReadBulk_AddsAndAdvisesEverySnapshotTagBeforeTearingAnyDown()
{
RecordingMxAccessServer server = new();
using MxAccessSession session = MxAccessSession.CreateForTesting(server, new NoopEventSink());
int serverHandle = session.Register("client");
session.ReadBulk(serverHandle, new[] { "Tag.A", "Tag.B", "Tag.C" }, TimeSpan.FromMilliseconds(50), () => { });
List<string> lifecycle = server.Operations.Where(op => !op.StartsWith("Register", StringComparison.Ordinal)).ToList();
int lastAdvise = lifecycle.FindLastIndex(op => op.StartsWith("Advise:", StringComparison.Ordinal));
int firstUnAdvise = lifecycle.FindIndex(op => op.StartsWith("UnAdvise:", StringComparison.Ordinal));
int firstRemove = lifecycle.FindIndex(op => op.StartsWith("RemoveItem:", StringComparison.Ordinal));
Assert.Equal(3, lifecycle.Count(op => op.StartsWith("AddItem:", StringComparison.Ordinal)));
Assert.Equal(3, lifecycle.Count(op => op.StartsWith("Advise:", StringComparison.Ordinal)));
Assert.True(lastAdvise < firstUnAdvise, string.Join(",", lifecycle));
Assert.True(lastAdvise < firstRemove, string.Join(",", lifecycle));
}
/// <summary>
/// Every per-tag outcome keeps its exact pre-batching shape and request
/// order in one mixed request: blank address, cached-and-advised fast
/// path, a tag that delivers, an AddItem failure, an Advise failure, and
/// a tag that never delivers. A delivering tag returns as soon as its
/// value lands rather than waiting out the timeout for the others.
/// </summary>
[Fact]
public void ReadBulk_MixedOutcomes_PreservesPerTagResultsAndOrder()
{
RecordingMxAccessServer server = new();
server.FailAddItemFor.Add("Tag.AddFails");
server.FailAdviseFor.Add("Tag.AdviseFails");
MxAccessValueCache valueCache = new();
using MxAccessSession session = MxAccessSession.CreateForTesting(server, new NoopEventSink(), valueCache: valueCache);
int serverHandle = session.Register("client");
// Caller-owned advised tag with a cached value: served from the cache.
int cachedHandle = session.AddItem(serverHandle, "Tag.Cached");
session.Advise(serverHandle, cachedHandle);
valueCache.Set(serverHandle, cachedHandle, BuildDataChange(serverHandle, cachedHandle, 11));
int operationsBeforeRead = server.Operations.Count;
// Pump step: deliver Tag.Delivers' first OnDataChange once advised.
Action pumpStep = () =>
{
if (server.TryGetAdvisedHandle("Tag.Delivers", out int handle)
&& !valueCache.TryGet(serverHandle, handle, out _))
{
valueCache.Set(serverHandle, handle, BuildDataChange(serverHandle, handle, 42));
}
};
string[] tags = { " ", "Tag.Cached", "Tag.Delivers", "Tag.AddFails", "Tag.AdviseFails", "Tag.Silent" };
IReadOnlyList<BulkReadResult> results = session.ReadBulk(serverHandle, tags, TimeSpan.FromMilliseconds(150), pumpStep);
Assert.Equal(tags.Length, results.Count);
for (int index = 0; index < tags.Length; index++)
{
Assert.Equal(tags[index], results[index].TagAddress);
}
Assert.False(results[0].WasSuccessful);
Assert.Equal(0, results[0].ItemHandle);
Assert.Equal("Tag address is required.", results[0].ErrorMessage);
Assert.True(results[1].WasSuccessful);
Assert.True(results[1].WasCached);
Assert.Equal(cachedHandle, results[1].ItemHandle);
Assert.Equal(11, results[1].Value.Int32Value);
Assert.True(results[2].WasSuccessful);
Assert.False(results[2].WasCached);
Assert.Equal(server.HandleForTag("Tag.Delivers"), results[2].ItemHandle);
Assert.Equal(42, results[2].Value.Int32Value);
Assert.Equal(192, results[2].Quality);
Assert.False(results[3].WasSuccessful);
Assert.Equal(0, results[3].ItemHandle);
Assert.Equal("AddItem failed for Tag.AddFails", results[3].ErrorMessage);
Assert.False(results[4].WasSuccessful);
Assert.Equal(server.HandleForTag("Tag.AdviseFails"), results[4].ItemHandle);
Assert.Equal("Advise failed for Tag.AdviseFails", results[4].ErrorMessage);
Assert.False(results[5].WasSuccessful);
Assert.Equal(server.HandleForTag("Tag.Silent"), results[5].ItemHandle);
Assert.Contains(ReadBulkTimeoutText, results[5].ErrorMessage, StringComparison.OrdinalIgnoreCase);
// The caller's subscription is untouched; the snapshot handles are gone
// (the Advise-failed one removed without an UnAdvise).
List<string> readOperations = server.Operations.Skip(operationsBeforeRead).ToList();
Assert.DoesNotContain($"UnAdvise:{serverHandle}:{cachedHandle}", readOperations);
Assert.DoesNotContain($"RemoveItem:{serverHandle}:{cachedHandle}", readOperations);
Assert.DoesNotContain($"UnAdvise:{serverHandle}:{server.HandleForTag("Tag.AdviseFails")}", readOperations);
Assert.Contains($"RemoveItem:{serverHandle}:{server.HandleForTag("Tag.AdviseFails")}", readOperations);
Assert.Contains($"UnAdvise:{serverHandle}:{server.HandleForTag("Tag.Delivers")}", readOperations);
Assert.Contains($"RemoveItem:{serverHandle}:{server.HandleForTag("Tag.Silent")}", readOperations);
RegisteredItemHandle remaining = Assert.Single(session.HandleRegistry.ItemHandles);
Assert.Equal(cachedHandle, remaining.ItemHandle);
}
/// <summary>
/// When every snapshot tag delivers, the wait ends as soon as the last
/// value lands instead of running out the deadline.
/// </summary>
[Fact]
public void ReadBulk_AllTagsDeliver_ReturnsWithoutWaitingOutTheTimeout()
{
RecordingMxAccessServer server = new();
MxAccessValueCache valueCache = new();
using MxAccessSession session = MxAccessSession.CreateForTesting(server, new NoopEventSink(), valueCache: valueCache);
int serverHandle = session.Register("client");
string[] tags = { "Tag.1", "Tag.2", "Tag.3" };
Action pumpStep = () =>
{
foreach (string tag in tags)
{
if (server.TryGetAdvisedHandle(tag, out int handle) && !valueCache.TryGet(serverHandle, handle, out _))
{
valueCache.Set(serverHandle, handle, BuildDataChange(serverHandle, handle, handle));
}
}
};
Stopwatch stopwatch = Stopwatch.StartNew();
IReadOnlyList<BulkReadResult> results = session.ReadBulk(serverHandle, tags, TimeSpan.FromSeconds(10), pumpStep);
stopwatch.Stop();
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(5), $"ReadBulk took {stopwatch.Elapsed.TotalMilliseconds:F0} ms.");
Assert.All(results, result => Assert.True(result.WasSuccessful));
for (int index = 0; index < tags.Length; index++)
{
Assert.Equal(results[index].ItemHandle, results[index].Value.Int32Value);
}
}
private static void RecordCompletion(MxAccessWriteCompletionCache cache, int serverHandle, int itemHandle)
{
RepeatedField<MxStatusProxy> statuses = new() { new MxStatusProxy { Category = MxStatusCategory.Ok } };
cache.Record(serverHandle, itemHandle, statuses);
}
private static MxEvent BuildDataChange(int serverHandle, int itemHandle, int value)
{
MxEvent mxEvent = new()
{
Family = MxEventFamily.OnDataChange,
ServerHandle = serverHandle,
ItemHandle = itemHandle,
Quality = 192,
SourceTimestamp = Timestamp.FromDateTime(new DateTime(2026, 10, 22, 0, 0, 0, DateTimeKind.Utc)),
Value = new MxValue
{
DataType = MxDataType.Integer,
VariantType = "VT_I4",
Int32Value = value,
},
OnDataChange = new OnDataChangeEvent(),
};
mxEvent.Statuses.Add(new MxStatusProxy { Category = MxStatusCategory.Ok });
return mxEvent;
}
/// <summary>Test sink that owns a real completion cache (Attach/Detach are no-ops).</summary>
private sealed class CompletionCacheSink : IMxAccessEventSink, IWriteCompletionCacheProvider
{
/// <inheritdoc />
public MxAccessWriteCompletionCache WriteCompletionCache { get; } = new();
/// <inheritdoc />
public void Attach(object mxAccessComObject, string sessionId)
{
}
/// <inheritdoc />
public void Detach()
{
}
}
/// <summary>
/// Fake server that hands out increasing handles, records every
/// lifecycle call, and can fail AddItem or Advise for named tags.
/// </summary>
private sealed class RecordingMxAccessServer : IMxAccessServer
{
private readonly NoopMxAccessServer noop = new();
private readonly Dictionary<string, int> handleByTag = new(StringComparer.Ordinal);
private readonly Dictionary<int, string> tagByHandle = new();
private readonly HashSet<int> advised = new();
private int nextServerHandle = 10;
private int nextItemHandle = 100;
public List<string> Operations { get; } = new();
public HashSet<string> FailAddItemFor { get; } = new(StringComparer.Ordinal);
public HashSet<string> FailAdviseFor { get; } = new(StringComparer.Ordinal);
public int HandleForTag(string tag) => handleByTag[tag];
public bool TryGetAdvisedHandle(string tag, out int handle)
{
return handleByTag.TryGetValue(tag, out handle) && advised.Contains(handle);
}
public int Register(string clientName)
{
Operations.Add($"Register:{clientName}");
return nextServerHandle++;
}
public void Unregister(int serverHandle) => Operations.Add($"Unregister:{serverHandle}");
public int AddItem(int serverHandle, string itemDefinition)
{
Operations.Add($"AddItem:{serverHandle}:{itemDefinition}");
if (FailAddItemFor.Contains(itemDefinition))
{
throw new InvalidOperationException($"AddItem failed for {itemDefinition}");
}
int handle = nextItemHandle++;
handleByTag[itemDefinition] = handle;
tagByHandle[handle] = itemDefinition;
return handle;
}
public int AddItem2(int serverHandle, string itemDefinition, string itemContext) => AddItem(serverHandle, itemDefinition);
public void RemoveItem(int serverHandle, int itemHandle) => Operations.Add($"RemoveItem:{serverHandle}:{itemHandle}");
public void Advise(int serverHandle, int itemHandle)
{
Operations.Add($"Advise:{serverHandle}:{itemHandle}");
if (tagByHandle.TryGetValue(itemHandle, out string? tag) && FailAdviseFor.Contains(tag))
{
throw new InvalidOperationException($"Advise failed for {tag}");
}
advised.Add(itemHandle);
}
public void UnAdvise(int serverHandle, int itemHandle)
{
Operations.Add($"UnAdvise:{serverHandle}:{itemHandle}");
advised.Remove(itemHandle);
}
public void AdviseSupervisory(int serverHandle, int itemHandle) => Advise(serverHandle, itemHandle);
public int AddBufferedItem(int serverHandle, string itemDefinition, string itemContext) => AddItem(serverHandle, itemDefinition);
public void SetBufferedUpdateInterval(int serverHandle, int updateIntervalMilliseconds) =>
noop.SetBufferedUpdateInterval(serverHandle, updateIntervalMilliseconds);
public object Suspend(int serverHandle, int itemHandle) => noop.Suspend(serverHandle, itemHandle);
public object Activate(int serverHandle, int itemHandle) => noop.Activate(serverHandle, itemHandle);
public void Write(int serverHandle, int itemHandle, object? value, int userId) =>
Operations.Add($"Write:{serverHandle}:{itemHandle}");
public void Write2(int serverHandle, int itemHandle, object? value, object? timestampValue, int userId) =>
Operations.Add($"Write2:{serverHandle}:{itemHandle}");
public void WriteSecured(int serverHandle, int itemHandle, int currentUserId, int verifierUserId, object? value) =>
Operations.Add($"WriteSecured:{serverHandle}:{itemHandle}");
public void WriteSecured2(int serverHandle, int itemHandle, int currentUserId, int verifierUserId, object? value, object? timestampValue) =>
Operations.Add($"WriteSecured2:{serverHandle}:{itemHandle}");
public int AuthenticateUser(int serverHandle, string verifyUser, string verifyUserPassword) => 0;
public int ArchestrAUserToId(int serverHandle, string userIdGuid) => 0;
}
}
@@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Threading;
@@ -573,6 +574,136 @@ public sealed class MxAccessStaSessionTests
"Expected StaCallInProgress to clear once the alarm poll returned.");
}
/// <summary>
/// A transient PollOnce failure must not fault the whole session: with a
/// failure threshold of 3, a poll pattern that never fails three times in
/// a row (fail, fail, succeed, ...) keeps the session healthy, and each
/// transient failure is logged with type/HRESULT only — never the
/// exception message.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RunAlarmPollLoop_TransientFailuresBelowThreshold_DoNotFaultSession()
{
FakeAlarmCommandHandler handler = new()
{
PollExceptionFactory = call => call % 3 == 0
? null
: new COMException("secret-tag-name GetXmlCurrentAlarms2 failed.", unchecked((int)0x80004005)),
};
ConcurrentQueue<string> warnings = new();
using StaRuntime runtime = CreateRuntime();
MxAccessEventQueue eventQueue = new();
using MxAccessStaSession session = new(
runtime,
new FakeMxAccessComObjectFactory(),
new FakeMxAccessEventSink(),
eventQueue,
(_eq, _affinity, _comFactory) => handler);
session.AlarmPollInterval = TimeSpan.FromMilliseconds(10);
session.AlarmPollFailureThreshold = 3;
session.AlarmPollWarningSink = warnings.Enqueue;
await session.StartAsync("session-1", workerProcessId: 1);
DateTime deadline = DateTime.UtcNow.AddSeconds(10);
while (handler.PollCount < 12 && !eventQueue.IsFaulted && DateTime.UtcNow < deadline)
{
await Task.Delay(10);
}
Assert.False(eventQueue.IsFaulted, "Transient alarm-poll failures below the threshold must not fault the session.");
Assert.True(handler.PollCount >= 12, $"Expected the poll loop to keep polling; saw {handler.PollCount} polls.");
Assert.NotEmpty(warnings);
Assert.All(warnings, line =>
{
Assert.Contains("event=AlarmPollFailed", line, StringComparison.Ordinal);
Assert.Contains("hresult=0x80004005", line, StringComparison.Ordinal);
Assert.DoesNotContain("secret-tag-name", line, StringComparison.Ordinal);
});
}
/// <summary>
/// A run of failures reaching the threshold with no success in between
/// faults the session exactly once and stops the loop: two warnings, then
/// the fault on the third consecutive failure, then no further polls.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RunAlarmPollLoop_ConsecutiveFailuresReachThreshold_FaultsAndStops()
{
FakeAlarmCommandHandler handler = new()
{
PollException = new COMException("GetXmlCurrentAlarms2 failed.", unchecked((int)0x80004005)),
};
ConcurrentQueue<string> warnings = new();
using StaRuntime runtime = CreateRuntime();
MxAccessEventQueue eventQueue = new();
using MxAccessStaSession session = new(
runtime,
new FakeMxAccessComObjectFactory(),
new FakeMxAccessEventSink(),
eventQueue,
(_eq, _affinity, _comFactory) => handler);
session.AlarmPollInterval = TimeSpan.FromMilliseconds(10);
session.AlarmPollFailureThreshold = 3;
session.AlarmPollWarningSink = warnings.Enqueue;
await session.StartAsync("session-1", workerProcessId: 1);
DateTime deadline = DateTime.UtcNow.AddSeconds(10);
while (!eventQueue.IsFaulted && DateTime.UtcNow < deadline)
{
await Task.Delay(10);
}
Assert.True(eventQueue.IsFaulted);
await Task.Delay(200);
Assert.Equal(3, handler.PollCount);
Assert.Equal(2, warnings.Count);
WorkerFault? fault = session.DrainFault();
Assert.NotNull(fault);
Assert.Equal(WorkerFaultCategory.MxaccessEventConversionFailed, fault!.Category);
}
/// <summary>
/// WRK-02: when the STA loop dies after the session started, the death is
/// recorded as a StaHung fault on the event queue — the path the IPC drain
/// loop turns into a WorkerFault frame and a terminated pipe session.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StaLoopDeath_AfterStart_RecordsStaHungFaultOnEventQueue()
{
ControllableMessagePump pump = new();
using StaRuntime runtime = new(new NoopComApartmentInitializer(), pump, TimeSpan.FromMilliseconds(25));
MxAccessEventQueue eventQueue = new();
using MxAccessStaSession session = new(
runtime,
new FakeMxAccessComObjectFactory(),
new FakeMxAccessEventSink(),
eventQueue);
await session.StartAsync("session-1", workerProcessId: 1);
Assert.False(eventQueue.IsFaulted);
pump.ThrowOnNextPump(new InvalidOperationException("The worker STA message pump failed."));
DateTime deadline = DateTime.UtcNow.AddSeconds(5);
while (!eventQueue.IsFaulted && DateTime.UtcNow < deadline)
{
await Task.Delay(10);
}
Assert.True(eventQueue.IsFaulted, "Expected the STA loop death to fault the event queue.");
WorkerFault? fault = session.DrainFault();
Assert.NotNull(fault);
Assert.Equal(WorkerFaultCategory.StaHung, fault!.Category);
Assert.Equal(typeof(InvalidOperationException).FullName, fault.ExceptionType);
Assert.Contains("STA loop terminated", fault.DiagnosticMessage, StringComparison.Ordinal);
Assert.Equal(ProtocolStatusCode.WorkerUnavailable, fault.ProtocolStatus.Code);
}
/// <summary>
/// The STA-affinity guard throws when an
/// IMxAccessAlarmConsumer call is attempted off the thread that created
@@ -621,6 +752,9 @@ public sealed class MxAccessStaSessionTests
/// <summary>Exception thrown by PollOnce; null to succeed.</summary>
public Exception? PollException { get; set; }
/// <summary>Per-call exception factory (1-based call number); null result succeeds.</summary>
public Func<int, Exception?>? PollExceptionFactory { get; set; }
/// <summary>When set, <see cref="PollOnce"/> blocks until <see cref="ReleasePoll"/> is called.</summary>
public bool BlockPoll { get; set; }
@@ -699,6 +833,22 @@ public sealed class MxAccessStaSessionTests
{
throw PollException;
}
Func<int, Exception?>? factory = PollExceptionFactory;
if (factory is not null)
{
int callNumber;
lock (gate)
{
callNumber = pollCount;
}
Exception? exception = factory(callNumber);
if (exception is not null)
{
throw exception;
}
}
}
/// <inheritdoc />
@@ -155,6 +155,124 @@ public sealed class StaRuntimeTests
Assert.Contains("shutting down", exception.Message);
}
/// <summary>
/// A backlog of queued work items must not starve inbound COM
/// callbacks: the loop pumps Windows messages between items rather
/// than only after draining the whole queue. A gate item holds the STA
/// while two more items queue behind it; once released, the second
/// queued item must observe a pump that happened after the first.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ProcessQueuedCommands_PumpsMessagesBetweenQueuedWorkItems()
{
ControllableMessagePump pump = new();
using StaRuntime runtime = new(new RecordingComApartmentInitializer(), pump, TimeSpan.FromSeconds(30));
runtime.Start();
using ManualResetEventSlim gateEntered = new(false);
using ManualResetEventSlim releaseGate = new(false);
Task gate = runtime.InvokeAsync(() =>
{
gateEntered.Set();
releaseGate.Wait(TimeSpan.FromSeconds(10));
});
Assert.True(gateEntered.Wait(TimeSpan.FromSeconds(5)));
Task<int> first = runtime.InvokeAsync(() => pump.PumpCount);
Task<int> second = runtime.InvokeAsync(() => pump.PumpCount);
releaseGate.Set();
await gate;
int pumpsSeenByFirst = await first;
int pumpsSeenBySecond = await second;
Assert.True(
pumpsSeenBySecond > pumpsSeenByFirst,
$"Expected a pump between queued work items (first saw {pumpsSeenByFirst}, second saw {pumpsSeenBySecond}).");
}
/// <summary>
/// WRK-02: an exception that kills the STA loop after a successful
/// <see cref="StaRuntime.Start"/> must be surfaced — recorded on
/// <see cref="StaRuntime.LoopFault"/>, handed to the registered fault
/// handler, and turned into fast failures for every later
/// <see cref="StaRuntime.InvokeAsync{T}(Func{T}, CancellationToken)"/> —
/// rather than parked in a startup field nothing reads.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task LoopException_AfterStart_IsReportedToHandlerAndFailsLaterCommands()
{
ControllableMessagePump pump = new();
RecordingComApartmentInitializer initializer = new();
using StaRuntime runtime = new(initializer, pump, TimeSpan.FromMilliseconds(25));
runtime.Start();
TaskCompletionSource<Exception> reported = new(TaskCreationOptions.RunContinuationsAsynchronously);
runtime.RegisterLoopFaultHandler(exception => reported.TrySetResult(exception));
Assert.Null(runtime.LoopFault);
InvalidOperationException loopException = new("pump failed");
pump.ThrowOnNextPump(loopException);
Task completed = await Task.WhenAny(reported.Task, Task.Delay(TimeSpan.FromSeconds(5)));
Assert.Same(reported.Task, completed);
Assert.Same(loopException, await reported.Task);
Assert.Same(loopException, runtime.LoopFault);
InvalidOperationException rejected = await Assert.ThrowsAsync<InvalidOperationException>(
() => runtime.InvokeAsync(() => 1));
Assert.IsNotType<StaRuntimeShutdownException>(rejected);
Assert.Same(loopException, rejected.InnerException);
// The dying thread still runs its apartment cleanup.
Assert.True(runtime.Shutdown(TimeSpan.FromSeconds(5)));
Assert.False(runtime.IsRunning);
Assert.Equal(1, initializer.UninitializeCount);
}
/// <summary>
/// A handler registered after the loop already died still receives the
/// fault, so the window between Start and registration loses nothing.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RegisterLoopFaultHandler_AfterLoopDied_InvokesHandlerImmediately()
{
ControllableMessagePump pump = new();
using StaRuntime runtime = new(new RecordingComApartmentInitializer(), pump, TimeSpan.FromMilliseconds(25));
runtime.Start();
InvalidOperationException loopException = new("pump failed");
pump.ThrowOnNextPump(loopException);
DateTime deadline = DateTime.UtcNow.AddSeconds(5);
while (runtime.LoopFault is null && DateTime.UtcNow < deadline)
{
await Task.Delay(10);
}
Exception? seen = null;
runtime.RegisterLoopFaultHandler(exception => seen = exception);
Assert.Same(loopException, seen);
}
/// <summary>A requested shutdown is not a loop fault.</summary>
[Fact]
public void Shutdown_DoesNotRecordLoopFault()
{
using StaRuntime runtime = new(new RecordingComApartmentInitializer(), new ControllableMessagePump(), TimeSpan.FromMilliseconds(25));
runtime.Start();
bool invoked = false;
runtime.RegisterLoopFaultHandler(_ => invoked = true);
Assert.True(runtime.Shutdown(TimeSpan.FromSeconds(5)));
Assert.Null(runtime.LoopFault);
Assert.False(invoked);
}
private static StaRuntime CreateRuntime(RecordingComApartmentInitializer initializer)
{
return new StaRuntime(
@@ -0,0 +1,47 @@
using System;
using System.Threading;
using ZB.MOM.WW.MxGateway.Worker.Sta;
namespace ZB.MOM.WW.MxGateway.Worker.Tests.TestSupport;
/// <summary>
/// Test <see cref="IStaMessagePump"/> that counts pump calls and can be armed
/// to throw from the next pump, so tests can observe the STA loop's pump
/// interleaving and drive a post-start loop death without a real Win32
/// message queue. The idle wait blocks on the command wake event exactly
/// like the production pump's <c>MsgWaitForMultipleObjectsEx</c> does.
/// </summary>
internal sealed class ControllableMessagePump : IStaMessagePump
{
private int pumpCount;
private Exception? throwOnNextPump;
/// <summary>Gets the number of <see cref="PumpPendingMessages"/> calls so far.</summary>
public int PumpCount => Volatile.Read(ref pumpCount);
/// <summary>Arms the pump to throw <paramref name="exception"/> from its next pump call.</summary>
/// <param name="exception">The exception to throw once.</param>
public void ThrowOnNextPump(Exception exception)
{
Volatile.Write(ref throwOnNextPump, exception);
}
/// <inheritdoc />
public void WaitForWorkOrMessages(WaitHandle commandWakeEvent, TimeSpan timeout)
{
commandWakeEvent.WaitOne(timeout);
}
/// <inheritdoc />
public int PumpPendingMessages()
{
Interlocked.Increment(ref pumpCount);
Exception? exception = Interlocked.Exchange(ref throwOnNextPump, null);
if (exception is not null)
{
throw exception;
}
return 0;
}
}