Complete Rust worker command coverage and bridge the alarm provider
This commit is contained in:
@@ -475,6 +475,269 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Exercises bulk operations and both snapshot read paths over gRPC.</summary>
|
||||
/// <returns>A task that represents the live test.</returns>
|
||||
[RustWorkerLiveFact]
|
||||
public async Task GatewayGrpc_WithRustWorker_BulkItemsAndReads()
|
||||
{
|
||||
const string first = "TestMachine_001.TestChangingInt";
|
||||
const string second = "TestMachine_002.TestChangingInt";
|
||||
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||||
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
|
||||
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
|
||||
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(60));
|
||||
CancellationToken token = deadline.Token;
|
||||
await app.StartAsync(token).ConfigureAwait(false);
|
||||
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
||||
MxAccessGateway.MxAccessGatewayClient client = new(channel);
|
||||
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
||||
{
|
||||
ClientSessionName = "rust-worker-bulk-items",
|
||||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||||
string id = opened.SessionId;
|
||||
|
||||
try
|
||||
{
|
||||
async Task<MxCommandReply> InvokeAsync(MxCommand command)
|
||||
{
|
||||
MxCommandReply reply = await client.InvokeAsync(new MxCommandRequest
|
||||
{
|
||||
SessionId = id, ClientCorrelationId = Guid.NewGuid().ToString("N"), Command = command,
|
||||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
|
||||
return reply;
|
||||
}
|
||||
|
||||
int server = (await InvokeAsync(CreateRegisterRequest(id).Command).ConfigureAwait(false)).Register.ServerHandle;
|
||||
MxCommandReply extended = await InvokeAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AddItem2,
|
||||
AddItem2 = new AddItem2Command { ServerHandle = server, ItemDefinition = "TestChangingInt", ItemContext = "TestMachine_003" },
|
||||
}).ConfigureAwait(false);
|
||||
Assert.True(extended.AddItem2.ItemHandle > 0);
|
||||
|
||||
MxCommandReply subscribed = await InvokeAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.SubscribeBulk,
|
||||
SubscribeBulk = new SubscribeBulkCommand { ServerHandle = server, TagAddresses = { first } },
|
||||
}).ConfigureAwait(false);
|
||||
SubscribeResult subscription = Assert.Single(subscribed.SubscribeBulk.Results);
|
||||
Assert.True(subscription.WasSuccessful, subscription.ErrorMessage);
|
||||
|
||||
// Observe the permanent advice before checking the worker's value cache.
|
||||
using (AsyncServerStreamingCall<MxEvent> events = client.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = id, EmitStreamAttached = true },
|
||||
cancellationToken: token))
|
||||
{
|
||||
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||||
{
|
||||
MxEvent change = events.ResponseStream.Current;
|
||||
if (change.Family == MxEventFamily.OnDataChange && change.ItemHandle == subscription.ItemHandle)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MxCommandReply read = await InvokeAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.ReadBulk,
|
||||
ReadBulk = new ReadBulkCommand { ServerHandle = server, TimeoutMs = 3000, TagAddresses = { first, second, second } },
|
||||
}).ConfigureAwait(false);
|
||||
Assert.Collection(read.ReadBulk.Results,
|
||||
cached => { Assert.True(cached.WasSuccessful, cached.ErrorMessage); Assert.True(cached.WasCached); },
|
||||
snapshot => { Assert.True(snapshot.WasSuccessful, snapshot.ErrorMessage); Assert.False(snapshot.WasCached); },
|
||||
duplicate =>
|
||||
{
|
||||
Assert.True(duplicate.WasSuccessful, duplicate.ErrorMessage);
|
||||
Assert.False(duplicate.WasCached);
|
||||
Assert.Equal(read.ReadBulk.Results[1].ItemHandle, duplicate.ItemHandle);
|
||||
});
|
||||
|
||||
MxCommandReply added = await InvokeAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AddItemBulk,
|
||||
AddItemBulk = new AddItemBulkCommand { ServerHandle = server, TagAddresses = { second } },
|
||||
}).ConfigureAwait(false);
|
||||
SubscribeResult item = Assert.Single(added.AddItemBulk.Results);
|
||||
Assert.True(item.WasSuccessful, item.ErrorMessage);
|
||||
MxCommandReply advised = await InvokeAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AdviseItemBulk,
|
||||
AdviseItemBulk = new AdviseItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle } },
|
||||
}).ConfigureAwait(false);
|
||||
Assert.True(Assert.Single(advised.AdviseItemBulk.Results).WasSuccessful);
|
||||
MxCommandReply unadvised = await InvokeAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.UnAdviseItemBulk,
|
||||
UnAdviseItemBulk = new UnAdviseItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle } },
|
||||
}).ConfigureAwait(false);
|
||||
Assert.True(Assert.Single(unadvised.UnAdviseItemBulk.Results).WasSuccessful);
|
||||
MxCommandReply removed = await InvokeAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.RemoveItemBulk,
|
||||
RemoveItemBulk = new RemoveItemBulkCommand { ServerHandle = server, ItemHandles = { item.ItemHandle, extended.AddItem2.ItemHandle } },
|
||||
}).ConfigureAwait(false);
|
||||
Assert.All(removed.RemoveItemBulk.Results, result => Assert.True(result.WasSuccessful, result.ErrorMessage));
|
||||
MxCommandReply unsubscribed = await InvokeAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.UnsubscribeBulk,
|
||||
UnsubscribeBulk = new UnsubscribeBulkCommand { ServerHandle = server, ItemHandles = { subscription.ItemHandle } },
|
||||
}).ConfigureAwait(false);
|
||||
Assert.True(Assert.Single(unsubscribed.UnsubscribeBulk.Results).WasSuccessful);
|
||||
output.WriteLine("Bulk add, advise, cached and snapshot reads, unadvise, remove, subscribe, and unsubscribe passed.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
Assert.Equal(ProtocolStatusCode.Ok, (await client.CloseSessionAsync(new CloseSessionRequest
|
||||
{
|
||||
SessionId = id,
|
||||
}, deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false)).ProtocolStatus.Code);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Verifies managed alarm provider callbacks forwarded by the Rust worker.</summary>
|
||||
/// <returns>A task that represents the live test.</returns>
|
||||
[RustWorkerLiveFact]
|
||||
public async Task GatewayGrpc_WithRustWorker_ForwardsAlarmProviderCommandsAndEvents()
|
||||
{
|
||||
const string reference = "Galaxy!DEV.TestMachine_001.RustWorkerSmoke";
|
||||
string workerPath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
|
||||
await using GatewayServiceFixture fixture = new(workerPath, new TestWorkerProcessFactory(output), output);
|
||||
await using WebApplication app = BuildLoopbackGateway(fixture.Service);
|
||||
using CancellationTokenSource deadline = new(TimeSpan.FromSeconds(75));
|
||||
CancellationToken token = deadline.Token;
|
||||
await app.StartAsync(token).ConfigureAwait(false);
|
||||
using GrpcChannel channel = GrpcChannel.ForAddress(app.Urls.Single());
|
||||
MxAccessGateway.MxAccessGatewayClient client = new(channel);
|
||||
OpenSessionReply opened = await client.OpenSessionAsync(new OpenSessionRequest
|
||||
{
|
||||
ClientSessionName = "rust-worker-alarms",
|
||||
}, cancellationToken: token).ResponseAsync.ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, opened.ProtocolStatus.Code);
|
||||
string id = opened.SessionId;
|
||||
|
||||
try
|
||||
{
|
||||
async Task<MxCommandReply> AlarmAsync(MxCommand command)
|
||||
{
|
||||
WorkerCommandReply envelope = await fixture.Manager.InvokeAsync(id,
|
||||
new WorkerCommand { Command = command }, token).ConfigureAwait(false);
|
||||
Assert.Equal(command.Kind, envelope.Reply.Kind);
|
||||
return envelope.Reply;
|
||||
}
|
||||
|
||||
using AsyncServerStreamingCall<MxEvent> events = client.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = id, EmitStreamAttached = true }, cancellationToken: token);
|
||||
Assert.True(await events.ResponseStream.MoveNext(token).ConfigureAwait(false));
|
||||
Assert.NotNull(events.ResponseStream.Current.StreamAttached);
|
||||
MxCommandReply subscribed = await AlarmAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.SubscribeAlarms,
|
||||
SubscribeAlarms = new SubscribeAlarmsCommand
|
||||
{
|
||||
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!DEV",
|
||||
ForcedMode = AlarmProviderMode.Subtag,
|
||||
WatchList = { new AlarmSubtagTarget
|
||||
{
|
||||
AlarmFullReference = reference, SourceObjectReference = "TestMachine_001",
|
||||
ActiveSubtag = "TestMachine_001.TestChangingInt",
|
||||
} },
|
||||
},
|
||||
}).ConfigureAwait(false);
|
||||
Assert.True(subscribed.ProtocolStatus.Code == ProtocolStatusCode.Ok, subscribed.DiagnosticMessage);
|
||||
|
||||
MxEvent? transition = null;
|
||||
while (await events.ResponseStream.MoveNext(token).ConfigureAwait(false))
|
||||
{
|
||||
if (events.ResponseStream.Current.OnAlarmTransition?.AlarmFullReference == reference)
|
||||
{
|
||||
transition = events.ResponseStream.Current;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Assert.NotNull(transition);
|
||||
Assert.Equal(id, transition.SessionId);
|
||||
Assert.True(transition.WorkerSequence > 0);
|
||||
Assert.True(transition.OnAlarmTransition.Degraded);
|
||||
output.WriteLine($"Forwarded {transition.OnAlarmTransition.TransitionKind} at worker sequence {transition.WorkerSequence}.");
|
||||
|
||||
MxCommandReply snapshot = await AlarmAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.QueryActiveAlarms,
|
||||
QueryActiveAlarmsCommand = new QueryActiveAlarmsCommand { AlarmFilterPrefix = reference },
|
||||
}).ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, snapshot.ProtocolStatus.Code);
|
||||
Assert.Contains(snapshot.QueryActiveAlarms.Snapshots, alarm => alarm.AlarmFullReference == reference);
|
||||
MxCommandReply unknownAck = await AlarmAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AcknowledgeAlarm,
|
||||
AcknowledgeAlarmCommand = new AcknowledgeAlarmCommand { AlarmGuid = Guid.NewGuid().ToString() },
|
||||
}).ConfigureAwait(false);
|
||||
Assert.NotNull(unknownAck.AcknowledgeAlarm);
|
||||
Assert.NotEqual(0, unknownAck.AcknowledgeAlarm.NativeStatus);
|
||||
MxCommandReply unknownName = await AlarmAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AcknowledgeAlarmByName,
|
||||
AcknowledgeAlarmByNameCommand = new AcknowledgeAlarmByNameCommand
|
||||
{
|
||||
AlarmName = "RustWorkerMissingAlarm", ProviderName = "Galaxy", GroupName = "DEV",
|
||||
},
|
||||
}).ConfigureAwait(false);
|
||||
Assert.NotNull(unknownName.AcknowledgeAlarm);
|
||||
Assert.NotEqual(0, unknownName.AcknowledgeAlarm.NativeStatus);
|
||||
MxCommandReply update = await AlarmAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.UpdateAlarmWatchList,
|
||||
UpdateAlarmWatchList = new UpdateAlarmWatchListCommand(),
|
||||
}).ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.MxaccessFailure, update.ProtocolStatus.Code);
|
||||
MxCommandReply unsubscribed = await AlarmAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.UnsubscribeAlarms,
|
||||
UnsubscribeAlarms = new UnsubscribeAlarmsCommand(),
|
||||
}).ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, unsubscribed.ProtocolStatus.Code);
|
||||
|
||||
MxCommandReply fallback = await AlarmAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.SubscribeAlarms,
|
||||
SubscribeAlarms = new SubscribeAlarmsCommand
|
||||
{
|
||||
SubscriptionExpression = $@"\\{Environment.MachineName}\Galaxy!DEV",
|
||||
FallbackEnabled = true,
|
||||
Failover = new AlarmFailoverConfig
|
||||
{
|
||||
ConsecutiveFailureThreshold = 1, FailbackProbeIntervalSeconds = 30, FailbackStableProbes = 3,
|
||||
},
|
||||
},
|
||||
}).ConfigureAwait(false);
|
||||
Assert.True(fallback.ProtocolStatus.Code == ProtocolStatusCode.Ok, fallback.DiagnosticMessage);
|
||||
MxCommandReply replacement = await AlarmAsync(new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.UpdateAlarmWatchList,
|
||||
UpdateAlarmWatchList = new UpdateAlarmWatchListCommand
|
||||
{
|
||||
WatchList = { new AlarmSubtagTarget
|
||||
{
|
||||
AlarmFullReference = reference, SourceObjectReference = "TestMachine_001",
|
||||
ActiveSubtag = "TestMachine_001.TestChangingInt",
|
||||
} },
|
||||
},
|
||||
}).ConfigureAwait(false);
|
||||
Assert.True(replacement.ProtocolStatus.Code == ProtocolStatusCode.Ok, replacement.DiagnosticMessage);
|
||||
output.WriteLine("Updated the live alarm provider's fallback watch list.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
Assert.Equal(ProtocolStatusCode.Ok, (await client.CloseSessionAsync(new CloseSessionRequest
|
||||
{
|
||||
SessionId = id,
|
||||
}, deadline: DateTime.UtcNow.AddSeconds(25)).ResponseAsync.ConfigureAwait(false)).ProtocolStatus.Code);
|
||||
}
|
||||
}
|
||||
|
||||
private static WebApplication BuildLoopbackGateway(MxAccessGatewayService service)
|
||||
{
|
||||
WebApplicationBuilder builder = WebApplication.CreateSlimBuilder();
|
||||
@@ -2492,6 +2755,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
options,
|
||||
_metrics,
|
||||
logger: _loggerFactory.CreateLogger<SessionManager>());
|
||||
Manager = sessionManager;
|
||||
MxAccessGrpcMapper mapper = new();
|
||||
EventStreamService eventStreamService = new(
|
||||
sessionManager,
|
||||
@@ -2520,6 +2784,9 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
/// </summary>
|
||||
public MxAccessGatewayService Service { get; }
|
||||
|
||||
/// <summary>The manager used by the gateway's internal alarm monitor.</summary>
|
||||
public SessionManager Manager { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Looks up a session by id directly against the in-process registry. The abnormal
|
||||
/// worker-exit test needs to observe the session's State / FinalFault as the gateway
|
||||
|
||||
Reference in New Issue
Block a user