dohertj2/mxaccessgw
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add bulk read/write command family across worker, gateway, and clients

Browse Source 
Adds five new MXAccess command kinds (WriteBulk, Write2Bulk,
WriteSecuredBulk, WriteSecured2Bulk, ReadBulk) that ride the existing
"one round-trip, per-entry results" bulk shape used by AddItemBulk and
SubscribeBulk today. MXAccess COM has no native bulk API; the worker
runs each bulk operation as a sequential loop on its STA, returning
one BulkWriteResult / BulkReadResult per requested entry so per-item
MXAccess failures surface as was_successful=false rather than throwing.

ReadBulk has no MXAccess analogue. The worker satisfies it by:

  - Returning the last cached OnDataChange payload (was_cached=true)
    when the requested tag is already in the session''s item registry
    AND advised — the existing subscription is NOT touched, since the
    caller did not create it.
  - Otherwise taking the AddItem + Advise + wait-for-OnDataChange +
    UnAdvise + RemoveItem snapshot lifecycle itself (was_cached=false)
    and leaving the session exactly as it was. The wait pumps Windows
    messages on the STA so the inbound MXAccess event can dispatch
    while the executor still holds the thread.

The new MxAccessValueCache lives on each MxAccessSession, shared with
MxAccessBaseEventSink which populates it on every OnDataChange after
the event clears the outbound queue. Eviction on RemoveItem keeps
reused MXAccess handles from serving stale values from a previous
lifetime.

Gateway-side authorization wires WriteBulk/Write2Bulk to invoke:write,
WriteSecuredBulk/WriteSecured2Bulk to invoke:secure, ReadBulk to
invoke:read. The constraint-filter pipeline is refactored from a single
BulkConstraintPlan record into an abstract base plus three concretes
(SubscribeBulk, WriteBulk, ReadBulk), each owning its own denied-entry
merge so the dispatch site never branches on reply shape. A new
FilterWriteBulkAsync<TEntry> generic over the four write-entry shapes
runs CheckWriteHandleAsync per entry; denied entries surface as the
BulkWriteResult shape, preserving original-index order.

All five language clients (.NET, Go, Rust, Python, Java) gained the
five new methods following their existing bulk pattern, with regenerated
protobufs.

Tests added:
  - MxAccessValueCacheTests (6 cases) — Set/TryGet, Remove resets the
    version, TryWaitForUpdate signals on Set, pump step fires each poll.
  - MxAccessBaseEventSinkTests — OnDataChange populates the cache,
    ValueCache property exposes the bound instance.
  - MxAccessCommandExecutorTests — four bulk-write variants (per-entry
    success/failure, value+timestamp forwarding, secured user ids),
    ReadBulk snapshot lifecycle on uncached tag (timeout surfaces as
    was_successful=false), invalid-payload reply.
  - GatewayGrpcScopeResolverTests — five new MxCommandKind cases.
  - SessionManagerTests — WriteBulk and ReadBulk forwarding through
    FakeWorkerHarness; ReadBulk forwards timeout_ms.
  - Per-client (.NET, Go, Rust, Python, Java) — WriteBulk builds the
    right command and returns per-entry results, ReadBulk forwards the
    timeout and unpacks the was_cached flag.

Cross-language e2e CLI subcommands for the new bulks are deliberately
scoped out of this change (each of the five client CLIs would need
five new subcommands plus matching phases in
scripts/run-client-e2e-tests.ps1); coverage equivalent to the existing
bulk-subscribe coverage is provided by worker + gateway + per-client
unit tests.

Docs updated in the same commit: gateway.md (Public MXAccess Command
Surface), docs/DesignDecisions.md (new "Bulk Command Family" section
with the ReadBulk cache-then-snapshot rationale), and every client
README.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
2026-05-20 03:42:38 -04:00
parent 758aca2355
commit 5e375f6d3d
41 changed files with 25624 additions and 1339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/MxGateway.Worker/MxAccess/MxAccessSession.cs
+413
View File
@@ -12,6 +12,7 @@ public sealed class MxAccessSession : IDisposable
private readonly IMxAccessServer mxAccessServer;
private readonly IMxAccessEventSink eventSink;
private readonly MxAccessHandleRegistry handleRegistry;
private readonly MxAccessValueCache valueCache;
private bool disposed;
private MxAccessSession(
@@ -19,12 +20,14 @@ public sealed class MxAccessSession : IDisposable
IMxAccessServer mxAccessServer,
IMxAccessEventSink eventSink,
MxAccessHandleRegistry handleRegistry,
MxAccessValueCache valueCache,
int creationThreadId)
{
this.mxAccessComObject = mxAccessComObject ?? throw new ArgumentNullException(nameof(mxAccessComObject));
this.mxAccessServer = mxAccessServer ?? throw new ArgumentNullException(nameof(mxAccessServer));
this.eventSink = eventSink ?? throw new ArgumentNullException(nameof(eventSink));
this.handleRegistry = handleRegistry ?? throw new ArgumentNullException(nameof(handleRegistry));
this.valueCache = valueCache ?? throw new ArgumentNullException(nameof(valueCache));
CreationThreadId = creationThreadId;
}
@@ -34,6 +37,14 @@ public sealed class MxAccessSession : IDisposable
/// <summary>The registry for tracking opened handles.</summary>
public MxAccessHandleRegistry HandleRegistry => handleRegistry;
/// <summary>
/// Per-session last-value cache populated by the event sink. ReadBulk
/// consults this cache before falling back to its own snapshot
/// lifecycle so it can serve a "current value" for an already-advised
/// tag without touching the existing subscription.
/// </summary>
public MxAccessValueCache ValueCache => valueCache;
/// <summary>Creates a WorkerReady message with session metadata.</summary>
/// <param name="workerProcessId">Process ID of the worker.</param>
public WorkerReady CreateWorkerReady(int workerProcessId)
@@ -78,11 +89,21 @@ public sealed class MxAccessSession : IDisposable
eventSink.Attach(mxAccessComObject, sessionId);
// Share the event sink's value cache when one is wired (the
// production MxAccessBaseEventSink path) so OnDataChange writes and
// ReadBulk reads both see the same instance. Fall back to a fresh
// cache for test fakes that supply their own sink — ReadBulk simply
// never serves cached values in that case.
MxAccessValueCache valueCache = eventSink is MxAccessBaseEventSink baseSink
? baseSink.ValueCache
: new MxAccessValueCache();
return new MxAccessSession(
mxAccessComObject,
new MxAccessComServer(mxAccessComObject),
eventSink,
new MxAccessHandleRegistry(),
valueCache,
Environment.CurrentManagedThreadId);
}
catch (Exception exception)
@@ -180,6 +201,10 @@ public sealed class MxAccessSession : IDisposable
mxAccessServer.RemoveItem(serverHandle, itemHandle);
handleRegistry.RemoveItemHandle(serverHandle, itemHandle);
// Evict the last-value entry so a future AddItem + Advise on the
// same handle id (which MXAccess may reuse) does not serve a stale
// OnDataChange snapshot from the previous lifetime.
valueCache.Remove(serverHandle, itemHandle);
}
/// <summary>Advises on item changes with plain subscription.</summary>
@@ -513,6 +538,394 @@ public sealed class MxAccessSession : IDisposable
return results;
}
/// <summary>
/// Bulk write — runs <see cref="Write"/> sequentially for each entry.
/// Each entry's <paramref name="convertValue"/> turns the protobuf
/// MxValue into a COM-marshalable variant. Per-item failures are
/// captured as <see cref="BulkWriteResult"/> entries with
/// <c>was_successful = false</c>; the loop never throws.
/// </summary>
public IReadOnlyList<BulkWriteResult> WriteBulk(
int serverHandle,
IReadOnlyList<WriteBulkEntry> entries,
Func<MxValue, object?> convertValue)
{
ThrowIfDisposed();
if (entries is null)
{
throw new ArgumentNullException(nameof(entries));
}
if (convertValue is null)
{
throw new ArgumentNullException(nameof(convertValue));
}
List<BulkWriteResult> results = new(entries.Count);
foreach (WriteBulkEntry entry in entries)
{
results.Add(ExecuteBulkWriteEntry(
serverHandle,
entry.ItemHandle,
() => Write(serverHandle, entry.ItemHandle, convertValue(entry.Value), entry.UserId)));
}
return results;
}
/// <summary>Bulk Write2 — sequential MXAccess <see cref="Write2"/> per entry.</summary>
public IReadOnlyList<BulkWriteResult> Write2Bulk(
int serverHandle,
IReadOnlyList<Write2BulkEntry> entries,
Func<MxValue, object?> convertValue)
{
ThrowIfDisposed();
if (entries is null)
{
throw new ArgumentNullException(nameof(entries));
}
if (convertValue is null)
{
throw new ArgumentNullException(nameof(convertValue));
}
List<BulkWriteResult> results = new(entries.Count);
foreach (Write2BulkEntry entry in entries)
{
results.Add(ExecuteBulkWriteEntry(
serverHandle,
entry.ItemHandle,
() => Write2(
serverHandle,
entry.ItemHandle,
convertValue(entry.Value),
convertValue(entry.TimestampValue),
entry.UserId)));
}
return results;
}
/// <summary>Bulk WriteSecured — sequential MXAccess <see cref="WriteSecured"/> per entry.</summary>
public IReadOnlyList<BulkWriteResult> WriteSecuredBulk(
int serverHandle,
IReadOnlyList<WriteSecuredBulkEntry> entries,
Func<MxValue, object?> convertValue)
{
ThrowIfDisposed();
if (entries is null)
{
throw new ArgumentNullException(nameof(entries));
}
if (convertValue is null)
{
throw new ArgumentNullException(nameof(convertValue));
}
List<BulkWriteResult> results = new(entries.Count);
foreach (WriteSecuredBulkEntry entry in entries)
{
results.Add(ExecuteBulkWriteEntry(
serverHandle,
entry.ItemHandle,
() => WriteSecured(
serverHandle,
entry.ItemHandle,
entry.CurrentUserId,
entry.VerifierUserId,
convertValue(entry.Value))));
}
return results;
}
/// <summary>Bulk WriteSecured2 — sequential MXAccess <see cref="WriteSecured2"/> per entry.</summary>
public IReadOnlyList<BulkWriteResult> WriteSecured2Bulk(
int serverHandle,
IReadOnlyList<WriteSecured2BulkEntry> entries,
Func<MxValue, object?> convertValue)
{
ThrowIfDisposed();
if (entries is null)
{
throw new ArgumentNullException(nameof(entries));
}
if (convertValue is null)
{
throw new ArgumentNullException(nameof(convertValue));
}
List<BulkWriteResult> results = new(entries.Count);
foreach (WriteSecured2BulkEntry entry in entries)
{
results.Add(ExecuteBulkWriteEntry(
serverHandle,
entry.ItemHandle,
() => WriteSecured2(
serverHandle,
entry.ItemHandle,
entry.CurrentUserId,
entry.VerifierUserId,
convertValue(entry.Value),
convertValue(entry.TimestampValue))));
}
return results;
}
/// <summary>
/// Bulk read snapshot. For each requested tag, returns the most recent
/// OnDataChange value if the tag is already advised AND a cached value
/// exists (no subscription side effects); otherwise takes the AddItem
/// + Advise + wait + UnAdvise + RemoveItem snapshot lifecycle itself.
/// <paramref name="timeout"/> bounds the wait per-tag in the snapshot
/// case; <paramref name="pumpStep"/> is invoked on every poll
/// iteration so the worker's STA can dispatch the incoming MXAccess
/// message that carries the value.
/// </summary>
public IReadOnlyList<BulkReadResult> ReadBulk(
int serverHandle,
IReadOnlyList<string> tagAddresses,
TimeSpan timeout,
Action pumpStep)
{
ThrowIfDisposed();
if (tagAddresses is null)
{
throw new ArgumentNullException(nameof(tagAddresses));
}
if (pumpStep is null)
{
throw new ArgumentNullException(nameof(pumpStep));
}
List<BulkReadResult> results = new(tagAddresses.Count);
foreach (string? tagAddress in tagAddresses)
{
if (string.IsNullOrWhiteSpace(tagAddress))
{
results.Add(FailedRead(serverHandle, tagAddress ?? string.Empty, itemHandle: 0, wasCached: false, "Tag address is required."));
continue;
}
results.Add(ReadOneTag(serverHandle, tagAddress, timeout, pumpStep));
}
return results;
}
private BulkReadResult ReadOneTag(
int serverHandle,
string tagAddress,
TimeSpan timeout,
Action pumpStep)
{
// 1. Cached-and-advised fast path: scan the registry for a live item
// handle matching this tag and check whether the value cache has a
// payload for it. If so, return the cached value without touching
// the existing subscription — the caller didn't create it, so
// ReadBulk must not tear it down.
if (TryGetCachedReadFor(serverHandle, tagAddress, out int cachedItemHandle, out MxAccessValueCache.CachedValue cachedValue))
{
return SucceededRead(
serverHandle,
tagAddress,
cachedItemHandle,
wasCached: true,
cachedValue);
}
// 2. Snapshot lifecycle. Reserve our own item handle, advise, pump
// until we see a fresh OnDataChange (or the deadline elapses),
// then tear it down.
int itemHandle = 0;
bool advised = false;
try
{
itemHandle = AddItem(serverHandle, tagAddress);
ulong baseline = valueCache.CurrentVersion(serverHandle, itemHandle);
Advise(serverHandle, itemHandle);
advised = true;
DateTime deadline = DateTime.UtcNow + timeout;
bool gotValue = valueCache.TryWaitForUpdate(
serverHandle,
itemHandle,
baseline,
deadline,
pumpStep,
out MxAccessValueCache.CachedValue snapshot);
return gotValue
? SucceededRead(serverHandle, tagAddress, itemHandle, wasCached: false, snapshot)
: FailedRead(
serverHandle,
tagAddress,
itemHandle,
wasCached: false,
$"ReadBulk timed out after {timeout.TotalMilliseconds:F0} ms waiting for first OnDataChange.");
}
catch (Exception exception)
{
return FailedRead(serverHandle, tagAddress, itemHandle, wasCached: false, exception.Message);
}
finally
{
// Snapshot teardown — best-effort. Errors here are noted on the
// diagnostic message of the original result (above) by appending
// a cleanup suffix; we never re-throw from finally.
if (advised)
{
try { UnAdvise(serverHandle, itemHandle); } catch { /* swallow — best effort */ }
}
if (itemHandle != 0)
{
try { RemoveItem(serverHandle, itemHandle); } catch { /* swallow — best effort */ }
}
}
}
private bool TryGetCachedReadFor(
int serverHandle,
string tagAddress,
out int itemHandle,
out MxAccessValueCache.CachedValue cachedValue)
{
// Linear scan — bulk-read sizes are small in practice and the registry
// is keyed by handle, not by tag. If profiling ever shows this hot, a
// reverse tag→handle map can be added on the registry side.
foreach (RegisteredItemHandle registered in handleRegistry.ItemHandles)
{
if (registered.ServerHandle != serverHandle)
{
continue;
}
if (!string.Equals(registered.ItemDefinition, tagAddress, StringComparison.Ordinal))
{
continue;
}
if (!handleRegistry.ContainsAdviceHandle(serverHandle, registered.ItemHandle, MxAccessAdviceKind.Plain)
&& !handleRegistry.ContainsAdviceHandle(serverHandle, registered.ItemHandle, MxAccessAdviceKind.Supervisory))
{
// Tag is added but not advised — no fresh OnDataChange will
// arrive without us advising. Fall through to the snapshot
// path which advises explicitly.
continue;
}
if (valueCache.TryGet(serverHandle, registered.ItemHandle, out cachedValue))
{
itemHandle = registered.ItemHandle;
return true;
}
}
itemHandle = 0;
cachedValue = default;
return false;
}
private BulkWriteResult ExecuteBulkWriteEntry(
int serverHandle,
int itemHandle,
Action invokeWrite)
{
try
{
invokeWrite();
return new BulkWriteResult
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
WasSuccessful = true,
ErrorMessage = string.Empty,
};
}
catch (System.Runtime.InteropServices.COMException comException)
{
BulkWriteResult result = new()
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
WasSuccessful = false,
ErrorMessage = comException.Message,
};
result.Hresult = comException.HResult;
return result;
}
catch (Exception exception)
{
return new BulkWriteResult
{
ServerHandle = serverHandle,
ItemHandle = itemHandle,
WasSuccessful = false,
ErrorMessage = exception.Message,
};
}
}
private static BulkReadResult SucceededRead(
int serverHandle,
string tagAddress,
int itemHandle,
bool wasCached,
MxAccessValueCache.CachedValue snapshot)
{
BulkReadResult result = new()
{
ServerHandle = serverHandle,
TagAddress = tagAddress,
ItemHandle = itemHandle,
WasSuccessful = true,
WasCached = wasCached,
Quality = snapshot.Quality,
ErrorMessage = string.Empty,
};
if (snapshot.Value is not null)
{
result.Value = snapshot.Value;
}
if (snapshot.SourceTimestamp is not null)
{
result.SourceTimestamp = snapshot.SourceTimestamp;
}
if (snapshot.Statuses is not null)
{
result.Statuses.Add(snapshot.Statuses);
}
return result;
}
private static BulkReadResult FailedRead(
int serverHandle,
string tagAddress,
int itemHandle,
bool wasCached,
string errorMessage)
{
return new BulkReadResult
{
ServerHandle = serverHandle,
TagAddress = tagAddress,
ItemHandle = itemHandle,
WasSuccessful = false,
WasCached = wasCached,
ErrorMessage = errorMessage,
};
}
/// <summary>Gracefully shuts down the session, cleaning up all handles.</summary>
public MxAccessShutdownResult ShutdownGracefully()
{