Complete Rust worker command coverage and bridge the alarm provider

This commit is contained in:
Joseph Doherty
2026-11-21 09:34:41 -05:00
parent d1769f81ff
commit dbc5ea0c25
12 changed files with 1945 additions and 160 deletions
@@ -475,6 +475,269 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
}
/// <summary>Exercises bulk operations and both snapshot read paths over gRPC.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_BulkItemsAndReads()
{
const string first = "TestMachine_001.TestChangingInt";
const string second = "TestMachine_002.TestChangingInt";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(60));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-worker-bulk-items",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string id = opened.SessionId;
try
{
async Task<MxCommandReply> InvokeAsync(MxCommand command)
{
MxCommandReply reply = await client.InvokeAsync(new MxCommandRequest
{
SessionId = id, ClientCorrelationId = Guid.NewGuid().ToString("N"), Command = command,
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
return reply;
}
int server = (await InvokeAsync(CreateRegisterRequest(id).Command).ConfigureAwait(false)).Register.ServerHandle;
MxCommandReply extended = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.AddItem2,
AddItem2 = new AddItem2Command { ServerHandle = server, ItemDefinition = "TestChangingInt", ItemContext = "TestMachine_003" },
}).ConfigureAwait(false);
Assert.True(extended.AddItem2.ItemHandle > 0);
MxCommandReply subscribed = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.SubscribeBulk,
SubscribeBulk = new SubscribeBulkCommand { ServerHandle = server, TagAddresses = { first } },
}).ConfigureAwait(false);
SubscribeResult subscription = Assert.Single(subscribed.SubscribeBulk.Results);
Assert.True(subscription.WasSuccessful, subscription.ErrorMessage);
// Observe the permanent advice before checking the worker's value cache.
using (AsyncServerStreamingCall<MxEvent> events = client.StreamEvents(
new StreamEventsRequest { SessionId = id, EmitStreamAttached = true },
cancellationToken: token))
{
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
MxEvent change = events.ResponseStream.Current;
if (change.Family == MxEventFamily.OnDataChange && change.ItemHandle == subscription.ItemHandle)
{
break;
}
}
}
MxCommandReply read = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.ReadBulk,
ReadBulk = new ReadBulkCommand { ServerHandle = server, TimeoutMs = 3000, TagAddresses = { first, second, second } },
}).ConfigureAwait(false);
Assert.Collection(read.ReadBulk.Results,
cached => { Assert.True(cached.WasSuccessful, cached.ErrorMessage); Assert.True(cached.WasCached); },
snapshot => { Assert.True(snapshot.WasSuccessful, snapshot.ErrorMessage); Assert.False(snapshot.WasCached); },
duplicate =>
{
Assert.True(duplicate.WasSuccessful, duplicate.ErrorMessage);
Assert.False(duplicate.WasCached);
Assert.Equal(read.ReadBulk.Results[1].ItemHandle, duplicate.ItemHandle);
});
MxCommandReply added = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.AddItemBulk,
AddItemBulk = new AddItemBulkCommand { ServerHandle = server, TagAddresses = { second } },
}).ConfigureAwait(false);
SubscribeResult item = Assert.Single(added.AddItemBulk.Results);
Assert.True(item.WasSuccessful, item.ErrorMessage);
MxCommandReply advised = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.AdviseItemBulk,
AdviseItemBulk = new AdviseItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle } },
}).ConfigureAwait(false);
Assert.True(Assert.Single(advised.AdviseItemBulk.Results).WasSuccessful);
MxCommandReply unadvised = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.UnAdviseItemBulk,
UnAdviseItemBulk = new UnAdviseItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle } },
}).ConfigureAwait(false);
Assert.True(Assert.Single(unadvised.UnAdviseItemBulk.Results).WasSuccessful);
MxCommandReply removed = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.RemoveItemBulk,
RemoveItemBulk = new RemoveItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle, extended.AddItem2.ItemHandle } },
}).ConfigureAwait(false);
Assert.All(removed.RemoveItemBulk.Results, result => Assert.True(result.WasSuccessful, result.ErrorMessage));
MxCommandReply unsubscribed = await InvokeAsync(new MxCommand
{
Kind = MxCommandKind.UnsubscribeBulk,
UnsubscribeBulk = new UnsubscribeBulkCommand { ServerHandle = server, ItemHandles = { subscription.ItemHandle } },
}).ConfigureAwait(false);
Assert.True(Assert.Single(unsubscribed.UnsubscribeBulk.Results).WasSuccessful);
output.WriteLine("Bulk add, advise, cached and snapshot reads, unadvise, remove, subscribe, and unsubscribe passed.");
}
finally
{
Assert.Equal(ProtocolStatusCode.Ok, (await client.CloseSessionAsync(new CloseSessionRequest
{
SessionId = id,
}, deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false)).ProtocolStatus.Code);
}
}
/// <summary>Verifies managed alarm provider callbacks forwarded by the Rust worker.</summary>
/// <returns>A task that represents the live test.</returns>
[RustWorkerLiveFact]
public async Task GatewayGrpc_WithRustWorker_ForwardsAlarmProviderCommandsAndEvents()
{
const string reference = "Galaxy!DEV.TestMachine_001.RustWorkerSmoke";
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(75));
CancellationToken token = deadline.Token;
await app.StartAsync(token).ConfigureAwait(false);
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
MxAccessGateway.MxAccessGatewayClient client = new(channel);
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
{
ClientSessionName = "rust-worker-alarms",
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
string id = opened.SessionId;
try
{
async Task<MxCommandReply> AlarmAsync(MxCommand command)
{
WorkerCommandReply envelope = await fixture.Manager.InvokeAsync(id,
new WorkerCommand { Command = command }, token).ConfigureAwait(false);
Assert.Equal(command.Kind, envelope.Reply.Kind);
return envelope.Reply;
}
using AsyncServerStreamingCall<MxEvent> events = client.StreamEvents(
new StreamEventsRequest { SessionId = id, EmitStreamAttached = true }, cancellationToken: token);
Assert.True(await events.ResponseStream.MoveNext(token).ConfigureAwait(false));
Assert.NotNull(events.ResponseStream.Current.StreamAttached);
MxCommandReply subscribed = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.SubscribeAlarms,
SubscribeAlarms = new SubscribeAlarmsCommand
{
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!DEV",
ForcedMode = AlarmProviderMode.Subtag,
WatchList = { new AlarmSubtagTarget
{
AlarmFullReference = reference, SourceObjectReference = "TestMachine_001",
ActiveSubtag = "TestMachine_001.TestChangingInt",
} },
},
}).ConfigureAwait(false);
Assert.True(subscribed.ProtocolStatus.Code == ProtocolStatusCode.Ok, subscribed.DiagnosticMessage);
MxEvent? transition = null;
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
{
if (events.ResponseStream.Current.OnAlarmTransition?.AlarmFullReference == reference)
{
transition = events.ResponseStream.Current;
break;
}
}
Assert.NotNull(transition);
Assert.Equal(id, transition.SessionId);
Assert.True(transition.WorkerSequence > 0);
Assert.True(transition.OnAlarmTransition.Degraded);
output.WriteLine($"Forwarded {transition.OnAlarmTransition.TransitionKind} at worker sequence {transition.WorkerSequence}.");
MxCommandReply snapshot = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.QueryActiveAlarms,
QueryActiveAlarmsCommand = new QueryActiveAlarmsCommand { AlarmFilterPrefix = reference },
}).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, snapshot.ProtocolStatus.Code);
Assert.Contains(snapshot.QueryActiveAlarms.Snapshots, alarm => alarm.AlarmFullReference == reference);
MxCommandReply unknownAck = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.AcknowledgeAlarm,
AcknowledgeAlarmCommand = new AcknowledgeAlarmCommand { AlarmGuid = Guid.NewGuid().ToString() },
}).ConfigureAwait(false);
Assert.NotNull(unknownAck.AcknowledgeAlarm);
Assert.NotEqual(0, unknownAck.AcknowledgeAlarm.NativeStatus);
MxCommandReply unknownName = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.AcknowledgeAlarmByName,
AcknowledgeAlarmByNameCommand = new AcknowledgeAlarmByNameCommand
{
AlarmName = "RustWorkerMissingAlarm", ProviderName = "Galaxy", GroupName = "DEV",
},
}).ConfigureAwait(false);
Assert.NotNull(unknownName.AcknowledgeAlarm);
Assert.NotEqual(0, unknownName.AcknowledgeAlarm.NativeStatus);
MxCommandReply update = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.UpdateAlarmWatchList,
UpdateAlarmWatchList = new UpdateAlarmWatchListCommand(),
}).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.MxaccessFailure, update.ProtocolStatus.Code);
MxCommandReply unsubscribed = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.UnsubscribeAlarms,
UnsubscribeAlarms = new UnsubscribeAlarmsCommand(),
}).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, unsubscribed.ProtocolStatus.Code);
MxCommandReply fallback = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.SubscribeAlarms,
SubscribeAlarms = new SubscribeAlarmsCommand
{
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!DEV",
FallbackEnabled = true,
Failover = new AlarmFailoverConfig
{
ConsecutiveFailureThreshold = 1, FailbackProbeIntervalSeconds = 30, FailbackStableProbes = 3,
},
},
}).ConfigureAwait(false);
Assert.True(fallback.ProtocolStatus.Code == ProtocolStatusCode.Ok, fallback.DiagnosticMessage);
MxCommandReply replacement = await AlarmAsync(new MxCommand
{
Kind = MxCommandKind.UpdateAlarmWatchList,
UpdateAlarmWatchList = new UpdateAlarmWatchListCommand
{
WatchList = { new AlarmSubtagTarget
{
AlarmFullReference = reference, SourceObjectReference = "TestMachine_001",
ActiveSubtag = "TestMachine_001.TestChangingInt",
} },
},
}).ConfigureAwait(false);
Assert.True(replacement.ProtocolStatus.Code == ProtocolStatusCode.Ok, replacement.DiagnosticMessage);
output.WriteLine("Updated the live alarm provider's fallback watch list.");
}
finally
{
Assert.Equal(ProtocolStatusCode.Ok, (await client.CloseSessionAsync(new CloseSessionRequest
{
SessionId = id,
}, deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false)).ProtocolStatus.Code);
}
}
private static WebApplication BuildLoopbackGateway(MxAccessGatewayService service)
{
WebApplicationBuilder builder = WebApplication.CreateSlimBuilder();
@@ -2492,6 +2755,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
options,
_metrics,
logger: _loggerFactory.CreateLogger<SessionManager>());
Manager = sessionManager;
MxAccessGrpcMapper mapper = new();
EventStreamService eventStreamService = new(
sessionManager,
@@ -2520,6 +2784,9 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
/// </summary>
public MxAccessGatewayService Service { get; }
/// <summary>The manager used by the gateway's internal alarm monitor.</summary>
public SessionManager Manager { get; }
/// <summary>
/// Looks up a session by id directly against the in-process registry. The abnormal
/// worker-exit test needs to observe the session's State / FinalFault as the gateway