rename: prefix gateway projects/namespaces with ZB.MOM.WW + sln→slnx
Apply the ZB.MOM.WW. prefix to all gateway-side projects, folders,
.csproj/.sln contents, C# namespaces, using directives, generated proto
C# (csharp_namespace + checked-in generated files), InternalsVisibleTo
attributes, project-name string literals (LoadProject, .sln lookups,
worker exe paths, staticwebassets manifest), and the install/script/doc
references that point at any of the above. Migrate the solution from
.sln to .slnx via `dotnet sln migrate` and delete the old file.
External-runtime identifiers are intentionally NOT prefixed so external
configuration keeps working:
- GatewayMetrics.cs MeterName ("MxGateway.Server")
- DashboardAuthenticationDefaults Scheme/Policy ("MxGateway.Dashboard")
- GatewayRequestLoggingMiddleware logger category ("MxGateway.Request")
- StaRuntime thread name ("MxGateway.Worker.STA")
- appsettings.json root section "MxGateway" + env-var prefix
MxGateway__... and secret-name MxGateway:ApiKeyPepper
- C:\ProgramData\MxGateway\ data dir paths
Also fixes two tests that were not rename-related but became visible
while validating the rename:
- WorkerLiveMxAccessSmokeTests.ShutDownAsync: cancellation that the
gateway service correctly maps to RpcException(Cancelled) per gRPC
convention was being misclassified as a stream fault. Added a sibling
catch on RpcException with StatusCode.Cancelled.
- IntegrationTestEnvironment.ResolveRepositoryRoot: extracted IsRepositoryRoot
and made it accept either a .git marker OR a .sln/.slnx next to src/
so the worker-exe walker works in non-git working copies.
clients/proto/proto-inputs.json's protoRoot updated to point at
src/ZB.MOM.WW.MxGateway.Contracts/Protos.
Verified by `dotnet build` and a full `dotnet test` of the .slnx with
MXGATEWAY_RUN_LIVE_{MXACCESS,LDAP,GALAXY}_TESTS=1:
Tests: 472/472 pass
Worker.Tests: 280/280 pass (4 dev-rig [Fact(Skip=...)] skipped)
IntegrationTests: 18/18 pass
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
using System.Security.Claims;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
public sealed class DashboardApiKeyAuthorizationTests
|
||||
{
|
||||
[Fact]
|
||||
public void CanManage_AuthenticatedUserWithShortRequiredGroupClaim_ReturnsTrue()
|
||||
{
|
||||
DashboardApiKeyAuthorization authorization = CreateAuthorization();
|
||||
ClaimsPrincipal user = CreatePrincipal("GwAdmin");
|
||||
|
||||
Assert.True(authorization.CanManage(user));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanManage_AuthenticatedUserWithRequiredGroupDnClaim_ReturnsTrue()
|
||||
{
|
||||
DashboardApiKeyAuthorization authorization = CreateAuthorization();
|
||||
ClaimsPrincipal user = CreatePrincipal("ou=GwAdmin,ou=groups,dc=lmxopcua,dc=local");
|
||||
|
||||
Assert.True(authorization.CanManage(user));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanManage_AnonymousUser_ReturnsFalse()
|
||||
{
|
||||
DashboardApiKeyAuthorization authorization = CreateAuthorization();
|
||||
ClaimsPrincipal user = new(new ClaimsIdentity());
|
||||
|
||||
Assert.False(authorization.CanManage(user));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanManage_AuthenticatedUserWithoutRequiredGroup_ReturnsFalse()
|
||||
{
|
||||
DashboardApiKeyAuthorization authorization = CreateAuthorization();
|
||||
ClaimsPrincipal user = CreatePrincipal("ReadOnly");
|
||||
|
||||
Assert.False(authorization.CanManage(user));
|
||||
}
|
||||
|
||||
private static DashboardApiKeyAuthorization CreateAuthorization()
|
||||
{
|
||||
return new DashboardApiKeyAuthorization(Options.Create(new GatewayOptions
|
||||
{
|
||||
Ldap = new LdapOptions
|
||||
{
|
||||
RequiredGroup = "GwAdmin",
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
private static ClaimsPrincipal CreatePrincipal(string group)
|
||||
{
|
||||
ClaimsIdentity identity = new(
|
||||
[new Claim(DashboardAuthenticationDefaults.LdapGroupClaimType, group)],
|
||||
DashboardAuthenticationDefaults.AuthenticationScheme);
|
||||
|
||||
return new ClaimsPrincipal(identity);
|
||||
}
|
||||
}
|
||||
+264
@@ -0,0 +1,264 @@
|
||||
using System.Security.Claims;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
public sealed class DashboardApiKeyManagementServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task CreateAsync_UnauthorizedUser_DoesNotCallStore()
|
||||
{
|
||||
FakeApiKeyAdminStore adminStore = new();
|
||||
DashboardApiKeyManagementService service = CreateService(adminStore);
|
||||
|
||||
DashboardApiKeyManagementResult result = await service.CreateAsync(
|
||||
new ClaimsPrincipal(new ClaimsIdentity()),
|
||||
CreateRequest(),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.False(result.Succeeded);
|
||||
Assert.Equal(0, adminStore.CreateCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CreateAsync_AuthorizedUser_StoresHashOfSecretAndAudits()
|
||||
{
|
||||
FakeApiKeyAdminStore adminStore = new();
|
||||
FakeApiKeyAuditStore auditStore = new();
|
||||
FakeApiKeySecretHasher hasher = new();
|
||||
DashboardApiKeyManagementService service = CreateService(adminStore, auditStore, hasher);
|
||||
|
||||
DashboardApiKeyManagementResult result = await service.CreateAsync(
|
||||
CreateAuthorizedUser(),
|
||||
CreateRequest(),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.True(result.Succeeded);
|
||||
Assert.NotNull(result.ApiKey);
|
||||
Assert.StartsWith("mxgw_operator01_", result.ApiKey, StringComparison.Ordinal);
|
||||
string secret = result.ApiKey["mxgw_operator01_".Length..];
|
||||
Assert.Equal(secret, hasher.LastSecret);
|
||||
Assert.DoesNotContain("mxgw_operator01_", hasher.LastSecret, StringComparison.Ordinal);
|
||||
ApiKeyCreateRequest stored = Assert.Single(adminStore.CreatedRequests);
|
||||
Assert.Equal("operator01", stored.KeyId);
|
||||
Assert.Equal("Operator", stored.DisplayName);
|
||||
Assert.Contains(GatewayScopes.SessionOpen, stored.Scopes);
|
||||
Assert.Equal(["Area1/*"], stored.Constraints.BrowseSubtrees);
|
||||
Assert.Contains(auditStore.Entries, entry =>
|
||||
entry.EventType == "dashboard-create-key"
|
||||
&& entry.KeyId == "operator01");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RevokeAsync_UnauthorizedUser_DoesNotCallStore()
|
||||
{
|
||||
FakeApiKeyAdminStore adminStore = new();
|
||||
DashboardApiKeyManagementService service = CreateService(adminStore);
|
||||
|
||||
DashboardApiKeyManagementResult result = await service.RevokeAsync(
|
||||
new ClaimsPrincipal(new ClaimsIdentity()),
|
||||
"operator01",
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.False(result.Succeeded);
|
||||
Assert.Equal(0, adminStore.RevokeCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RevokeAsync_AuthorizedUser_RevokesAndAudits()
|
||||
{
|
||||
FakeApiKeyAdminStore adminStore = new() { RevokeResult = true };
|
||||
FakeApiKeyAuditStore auditStore = new();
|
||||
DashboardApiKeyManagementService service = CreateService(adminStore, auditStore);
|
||||
|
||||
DashboardApiKeyManagementResult result = await service.RevokeAsync(
|
||||
CreateAuthorizedUser(),
|
||||
"operator01",
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.True(result.Succeeded);
|
||||
Assert.Equal("operator01", adminStore.LastRevokedKeyId);
|
||||
Assert.Contains(auditStore.Entries, entry =>
|
||||
entry.EventType == "dashboard-revoke-key"
|
||||
&& entry.KeyId == "operator01"
|
||||
&& entry.Details == "revoked");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RotateAsync_AuthorizedUser_RotatesHashAndAudits()
|
||||
{
|
||||
FakeApiKeyAdminStore adminStore = new() { RotateResult = true };
|
||||
FakeApiKeyAuditStore auditStore = new();
|
||||
FakeApiKeySecretHasher hasher = new();
|
||||
DashboardApiKeyManagementService service = CreateService(adminStore, auditStore, hasher);
|
||||
|
||||
DashboardApiKeyManagementResult result = await service.RotateAsync(
|
||||
CreateAuthorizedUser(),
|
||||
"operator01",
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.True(result.Succeeded);
|
||||
Assert.NotNull(result.ApiKey);
|
||||
Assert.StartsWith("mxgw_operator01_", result.ApiKey, StringComparison.Ordinal);
|
||||
Assert.Equal(hasher.HashSecret(hasher.LastSecret!), adminStore.LastRotatedSecretHash);
|
||||
Assert.Contains(auditStore.Entries, entry =>
|
||||
entry.EventType == "dashboard-rotate-key"
|
||||
&& entry.KeyId == "operator01"
|
||||
&& entry.Details == "rotated");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Server-004 regression: the dashboard create path must reject a request
|
||||
/// carrying a non-canonical scope string rather than persisting a key whose
|
||||
/// scope the authorization resolver never matches.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task CreateAsync_UnknownScope_DoesNotCallStore()
|
||||
{
|
||||
FakeApiKeyAdminStore adminStore = new();
|
||||
DashboardApiKeyManagementService service = CreateService(adminStore);
|
||||
|
||||
DashboardApiKeyManagementRequest request = CreateRequest() with
|
||||
{
|
||||
Scopes = new HashSet<string>(
|
||||
[GatewayScopes.SessionOpen, "invoke", "metadata"],
|
||||
StringComparer.Ordinal),
|
||||
};
|
||||
|
||||
DashboardApiKeyManagementResult result = await service.CreateAsync(
|
||||
CreateAuthorizedUser(),
|
||||
request,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.False(result.Succeeded);
|
||||
Assert.Equal(0, adminStore.CreateCount);
|
||||
}
|
||||
|
||||
private static DashboardApiKeyManagementService CreateService(
|
||||
FakeApiKeyAdminStore? adminStore = null,
|
||||
FakeApiKeyAuditStore? auditStore = null,
|
||||
FakeApiKeySecretHasher? hasher = null)
|
||||
{
|
||||
GatewayOptions options = new()
|
||||
{
|
||||
Ldap = new LdapOptions
|
||||
{
|
||||
RequiredGroup = "GwAdmin",
|
||||
},
|
||||
};
|
||||
|
||||
DefaultHttpContext httpContext = new();
|
||||
httpContext.Connection.RemoteIpAddress = System.Net.IPAddress.Loopback;
|
||||
|
||||
return new DashboardApiKeyManagementService(
|
||||
new DashboardApiKeyAuthorization(Options.Create(options)),
|
||||
adminStore ?? new FakeApiKeyAdminStore(),
|
||||
auditStore ?? new FakeApiKeyAuditStore(),
|
||||
hasher ?? new FakeApiKeySecretHasher(),
|
||||
new HttpContextAccessor { HttpContext = httpContext });
|
||||
}
|
||||
|
||||
private static DashboardApiKeyManagementRequest CreateRequest()
|
||||
{
|
||||
return new DashboardApiKeyManagementRequest(
|
||||
KeyId: "operator01",
|
||||
DisplayName: "Operator",
|
||||
Scopes: new HashSet<string>([GatewayScopes.SessionOpen], StringComparer.Ordinal),
|
||||
Constraints: ApiKeyConstraints.Empty with
|
||||
{
|
||||
BrowseSubtrees = ["Area1/*"],
|
||||
});
|
||||
}
|
||||
|
||||
private static ClaimsPrincipal CreateAuthorizedUser()
|
||||
{
|
||||
ClaimsIdentity identity = new(
|
||||
[new Claim(DashboardAuthenticationDefaults.LdapGroupClaimType, "GwAdmin")],
|
||||
DashboardAuthenticationDefaults.AuthenticationScheme);
|
||||
|
||||
return new ClaimsPrincipal(identity);
|
||||
}
|
||||
|
||||
private sealed class FakeApiKeyAdminStore : IApiKeyAdminStore
|
||||
{
|
||||
public int CreateCount { get; private set; }
|
||||
|
||||
public int RevokeCount { get; private set; }
|
||||
|
||||
public bool RevokeResult { get; init; }
|
||||
|
||||
public bool RotateResult { get; init; }
|
||||
|
||||
public string? LastRevokedKeyId { get; private set; }
|
||||
|
||||
public byte[]? LastRotatedSecretHash { get; private set; }
|
||||
|
||||
public List<ApiKeyCreateRequest> CreatedRequests { get; } = [];
|
||||
|
||||
public Task CreateAsync(ApiKeyCreateRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
CreateCount++;
|
||||
CreatedRequests.Add(request);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<ApiKeyRecord>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<ApiKeyRecord>>([]);
|
||||
}
|
||||
|
||||
public Task<bool> RevokeAsync(
|
||||
string keyId,
|
||||
DateTimeOffset revokedUtc,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
RevokeCount++;
|
||||
LastRevokedKeyId = keyId;
|
||||
return Task.FromResult(RevokeResult);
|
||||
}
|
||||
|
||||
public Task<bool> RotateAsync(
|
||||
string keyId,
|
||||
byte[] secretHash,
|
||||
DateTimeOffset rotatedUtc,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
LastRotatedSecretHash = secretHash;
|
||||
return Task.FromResult(RotateResult);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeApiKeyAuditStore : IApiKeyAuditStore
|
||||
{
|
||||
public List<ApiKeyAuditEntry> Entries { get; } = [];
|
||||
|
||||
public Task AppendAsync(ApiKeyAuditEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
Entries.Add(entry);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<ApiKeyAuditRecord>> ListRecentAsync(
|
||||
int count,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<ApiKeyAuditRecord>>([]);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeApiKeySecretHasher : IApiKeySecretHasher
|
||||
{
|
||||
public string? LastSecret { get; private set; }
|
||||
|
||||
public byte[] HashSecret(string secret)
|
||||
{
|
||||
LastSecret = secret;
|
||||
return System.Text.Encoding.UTF8.GetBytes($"hash:{secret}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
public sealed class DashboardAuthenticatorTests
|
||||
{
|
||||
[Fact]
|
||||
public void EscapeLdapFilter_EscapesSpecialCharacters()
|
||||
{
|
||||
string escaped = DashboardAuthenticator.EscapeLdapFilter("a\\b*c(d)e\0f");
|
||||
|
||||
Assert.Equal("a\\5cb\\2ac\\28d\\29e\\00f", escaped);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("GwAdmin", true)]
|
||||
[InlineData("gwadmin", true)]
|
||||
[InlineData("ou=GwAdmin,ou=groups,dc=lmxopcua,dc=local", true)]
|
||||
[InlineData("OtherGroup", false)]
|
||||
public void IsMemberOfRequiredGroup_MatchesShortNameAndDistinguishedName(
|
||||
string requiredGroup,
|
||||
bool expected)
|
||||
{
|
||||
string[] groups =
|
||||
[
|
||||
"ou=ReadOnly,ou=groups,dc=lmxopcua,dc=local",
|
||||
"ou=GwAdmin,ou=groups,dc=lmxopcua,dc=local"
|
||||
];
|
||||
|
||||
bool result = DashboardAuthenticator.IsMemberOfRequiredGroup(groups, requiredGroup);
|
||||
|
||||
Assert.Equal(expected, result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractFirstRdnValue_ReturnsLeadingRdnValue()
|
||||
{
|
||||
string result = DashboardAuthenticator.ExtractFirstRdnValue(
|
||||
"CN=Gateway Admins,OU=Groups,DC=example,DC=com");
|
||||
|
||||
Assert.Equal("Gateway Admins", result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AuthenticateAsync_LdapDisabled_ReturnsFailureWithoutRawCredentials()
|
||||
{
|
||||
DashboardAuthenticator authenticator = CreateAuthenticator(new GatewayOptions
|
||||
{
|
||||
Ldap = new LdapOptions
|
||||
{
|
||||
Enabled = false,
|
||||
},
|
||||
});
|
||||
|
||||
DashboardAuthenticationResult result = await authenticator.AuthenticateAsync(
|
||||
"admin",
|
||||
"admin123",
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.False(result.Succeeded);
|
||||
Assert.Null(result.Principal);
|
||||
Assert.DoesNotContain("admin123", result.FailureMessage, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private static DashboardAuthenticator CreateAuthenticator(GatewayOptions options)
|
||||
{
|
||||
return new DashboardAuthenticator(
|
||||
Options.Create(options),
|
||||
NullLogger<DashboardAuthenticator>.Instance);
|
||||
}
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
using System.Net;
|
||||
using System.Security.Claims;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
public sealed class DashboardAuthorizationHandlerTests
|
||||
{
|
||||
/// <summary>Verifies that unauthenticated remote requests fail authorization.</summary>
|
||||
[Fact]
|
||||
public async Task HandleAsync_UnauthenticatedRemoteRequest_DoesNotSucceed()
|
||||
{
|
||||
AuthorizationHandlerContext context = await AuthorizeAsync(
|
||||
new ClaimsPrincipal(new ClaimsIdentity()),
|
||||
IPAddress.Parse("10.0.0.5"),
|
||||
allowAnonymousLocalhost: false);
|
||||
|
||||
Assert.False(context.HasSucceeded);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that anonymous localhost access succeeds when allowed.</summary>
|
||||
[Fact]
|
||||
public async Task HandleAsync_AnonymousLocalhostAllowed_Succeeds()
|
||||
{
|
||||
AuthorizationHandlerContext context = await AuthorizeAsync(
|
||||
new ClaimsPrincipal(new ClaimsIdentity()),
|
||||
IPAddress.Loopback,
|
||||
allowAnonymousLocalhost: true);
|
||||
|
||||
Assert.True(context.HasSucceeded);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the anonymous-localhost bypass is denied when <c>AllowAnonymousLocalhost</c>
|
||||
/// is off, even on a loopback connection — the misconfiguration must not expose the dashboard.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task HandleAsync_AnonymousLocalhostDisallowed_DoesNotSucceed()
|
||||
{
|
||||
AuthorizationHandlerContext context = await AuthorizeAsync(
|
||||
new ClaimsPrincipal(new ClaimsIdentity()),
|
||||
IPAddress.Loopback,
|
||||
allowAnonymousLocalhost: false);
|
||||
|
||||
Assert.False(context.HasSucceeded);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the anonymous-localhost bypass stays scoped to loopback: an anonymous
|
||||
/// request from a non-loopback address is denied even when <c>AllowAnonymousLocalhost</c> is on.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task HandleAsync_AnonymousLocalhostAllowedFromRemoteAddress_DoesNotSucceed()
|
||||
{
|
||||
AuthorizationHandlerContext context = await AuthorizeAsync(
|
||||
new ClaimsPrincipal(new ClaimsIdentity()),
|
||||
IPAddress.Parse("10.0.0.5"),
|
||||
allowAnonymousLocalhost: true);
|
||||
|
||||
Assert.False(context.HasSucceeded);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that authenticated users without admin scope fail authorization.</summary>
|
||||
[Fact]
|
||||
public async Task HandleAsync_AuthenticatedWithoutAdminScope_DoesNotSucceed()
|
||||
{
|
||||
AuthorizationHandlerContext context = await AuthorizeAsync(
|
||||
CreatePrincipal(GatewayScopes.EventsRead),
|
||||
IPAddress.Loopback,
|
||||
allowAnonymousLocalhost: false);
|
||||
|
||||
Assert.False(context.HasSucceeded);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that authenticated users with admin scope succeed.</summary>
|
||||
[Fact]
|
||||
public async Task HandleAsync_AuthenticatedWithAdminScope_Succeeds()
|
||||
{
|
||||
AuthorizationHandlerContext context = await AuthorizeAsync(
|
||||
CreatePrincipal(GatewayScopes.Admin),
|
||||
IPAddress.Parse("10.0.0.5"),
|
||||
allowAnonymousLocalhost: false);
|
||||
|
||||
Assert.True(context.HasSucceeded);
|
||||
}
|
||||
|
||||
private static async Task<AuthorizationHandlerContext> AuthorizeAsync(
|
||||
ClaimsPrincipal principal,
|
||||
IPAddress remoteAddress,
|
||||
bool allowAnonymousLocalhost)
|
||||
{
|
||||
DashboardAuthorizationRequirement requirement = new();
|
||||
DefaultHttpContext httpContext = new();
|
||||
httpContext.Connection.RemoteIpAddress = remoteAddress;
|
||||
DashboardAuthorizationHandler handler = new(
|
||||
new HttpContextAccessor { HttpContext = httpContext },
|
||||
Options.Create(new GatewayOptions
|
||||
{
|
||||
Dashboard = new DashboardOptions
|
||||
{
|
||||
AllowAnonymousLocalhost = allowAnonymousLocalhost,
|
||||
RequireAdminScope = true
|
||||
}
|
||||
}));
|
||||
AuthorizationHandlerContext context = new([requirement], principal, httpContext);
|
||||
|
||||
await handler.HandleAsync(context);
|
||||
|
||||
return context;
|
||||
}
|
||||
|
||||
private static ClaimsPrincipal CreatePrincipal(string scope)
|
||||
{
|
||||
ClaimsIdentity identity = new(
|
||||
[new Claim(DashboardAuthenticationDefaults.ScopeClaimType, scope)],
|
||||
DashboardAuthenticationDefaults.AuthenticationScheme);
|
||||
|
||||
return new ClaimsPrincipal(identity);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for the pure projection/formatting helpers behind the
|
||||
/// dashboard Browse and Alarms tabs.
|
||||
/// </summary>
|
||||
public sealed class DashboardBrowseAndAlarmModelTests
|
||||
{
|
||||
[Fact]
|
||||
public void BuildTree_LinksChildrenToParents_AndPromotesOrphansToRoots()
|
||||
{
|
||||
GalaxyObject area = new() { GobjectId = 1, BrowseName = "AreaA", IsArea = true, ParentGobjectId = 0 };
|
||||
GalaxyObject child = new() { GobjectId = 2, BrowseName = "Pump01", ParentGobjectId = 1 };
|
||||
GalaxyObject orphan = new() { GobjectId = 3, BrowseName = "Lost", ParentGobjectId = 99 };
|
||||
|
||||
IReadOnlyList<DashboardBrowseNode> roots = DashboardBrowseTreeBuilder.Build([area, child, orphan]);
|
||||
|
||||
// The area and the orphan (its parent id is absent) are both roots.
|
||||
Assert.Equal(2, roots.Count);
|
||||
DashboardBrowseNode areaNode = Assert.Single(roots, node => node.Object.GobjectId == 1);
|
||||
Assert.Single(areaNode.Children);
|
||||
Assert.Equal(2, areaNode.Children[0].Object.GobjectId);
|
||||
Assert.Contains(roots, node => node.Object.GobjectId == 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTree_SortsAreasBeforeObjects()
|
||||
{
|
||||
GalaxyObject instance = new() { GobjectId = 1, BrowseName = "Zeta", IsArea = false };
|
||||
GalaxyObject areaB = new() { GobjectId = 2, BrowseName = "Beta", IsArea = true };
|
||||
|
||||
IReadOnlyList<DashboardBrowseNode> roots = DashboardBrowseTreeBuilder.Build([instance, areaB]);
|
||||
|
||||
Assert.Equal(2, roots.Count);
|
||||
Assert.True(roots[0].IsArea);
|
||||
Assert.Equal("Beta", roots[0].DisplayName);
|
||||
Assert.False(roots[1].IsArea);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true, "true")]
|
||||
[InlineData(false, "false")]
|
||||
public void FormatValue_FormatsBooleans(bool input, string expected)
|
||||
{
|
||||
MxValue value = new() { DataType = MxDataType.Boolean, BoolValue = input };
|
||||
Assert.Equal(expected, DashboardMxValueFormatter.FormatValue(value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FormatValue_FormatsNumbersAndStrings()
|
||||
{
|
||||
Assert.Equal("42", DashboardMxValueFormatter.FormatValue(new MxValue { Int32Value = 42 }));
|
||||
Assert.Equal("hello", DashboardMxValueFormatter.FormatValue(new MxValue { StringValue = "hello" }));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FormatValue_HandlesNullPayloadAndNullReference()
|
||||
{
|
||||
Assert.Equal("-", DashboardMxValueFormatter.FormatValue(null));
|
||||
Assert.Equal("(null)", DashboardMxValueFormatter.FormatValue(new MxValue { IsNull = true }));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TagValue_FromSuccessfulReadResult_MarksGoodQuality()
|
||||
{
|
||||
BulkReadResult result = new()
|
||||
{
|
||||
TagAddress = "Galaxy!Area.Tag",
|
||||
WasSuccessful = true,
|
||||
Quality = 192,
|
||||
Value = new MxValue { DataType = MxDataType.Double, DoubleValue = 1.5 },
|
||||
};
|
||||
|
||||
DashboardTagValue value = DashboardTagValue.FromBulkReadResult(result);
|
||||
|
||||
Assert.True(value.Ok);
|
||||
Assert.True(value.QualityGood);
|
||||
Assert.Equal("1.5", value.ValueText);
|
||||
Assert.Null(value.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TagValue_FromFailedReadResult_CarriesError()
|
||||
{
|
||||
BulkReadResult result = new()
|
||||
{
|
||||
TagAddress = "Galaxy!Area.Bad",
|
||||
WasSuccessful = false,
|
||||
Quality = 0,
|
||||
ErrorMessage = "invalid handle",
|
||||
};
|
||||
|
||||
DashboardTagValue value = DashboardTagValue.FromBulkReadResult(result);
|
||||
|
||||
Assert.False(value.Ok);
|
||||
Assert.False(value.QualityGood);
|
||||
Assert.Equal("invalid handle", value.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ActiveAlarm_FromSnapshot_ParsesProviderAndAcknowledgementState()
|
||||
{
|
||||
ActiveAlarmSnapshot unacked = new()
|
||||
{
|
||||
AlarmFullReference = "Galaxy!TestArea.TestMachine_001.TestAlarm001",
|
||||
Category = "TestArea",
|
||||
CurrentState = AlarmConditionState.Active,
|
||||
Severity = 500,
|
||||
};
|
||||
ActiveAlarmSnapshot acked = new()
|
||||
{
|
||||
AlarmFullReference = "Galaxy!TestArea.TestMachine_002.TestAlarm001",
|
||||
CurrentState = AlarmConditionState.ActiveAcked,
|
||||
};
|
||||
|
||||
DashboardActiveAlarm unackedRow = DashboardActiveAlarm.FromSnapshot(unacked);
|
||||
DashboardActiveAlarm ackedRow = DashboardActiveAlarm.FromSnapshot(acked);
|
||||
|
||||
Assert.Equal("Galaxy", unackedRow.Provider);
|
||||
Assert.Equal("TestArea", unackedRow.Area);
|
||||
Assert.Equal(500, unackedRow.Severity);
|
||||
Assert.True(unackedRow.IsUnacknowledged);
|
||||
Assert.False(ackedRow.IsUnacknowledged);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FormatValue_AndDataType_RenderArrayElementsAndElementType()
|
||||
{
|
||||
MxArray array = new() { ElementDataType = MxDataType.Double };
|
||||
array.Dimensions.Add(3u);
|
||||
array.DoubleValues = new DoubleArray();
|
||||
array.DoubleValues.Values.Add(new[] { 1.5, 2.25, 3.0 });
|
||||
MxValue value = new() { ArrayValue = array };
|
||||
|
||||
Assert.Equal("[1.5, 2.25, 3]", DashboardMxValueFormatter.FormatValue(value));
|
||||
Assert.Equal("Double[3]", DashboardMxValueFormatter.FormatDataType(value));
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
public sealed class DashboardConnectionStringDisplayTests
|
||||
{
|
||||
[Fact]
|
||||
public void GalaxyRepositoryConnectionString_WithSqlCredentials_OnlyKeepsNonSecretFields()
|
||||
{
|
||||
string display = DashboardConnectionStringDisplay.GalaxyRepositoryConnectionString(
|
||||
"Server=localhost;Database=ZB;User ID=mxuser;Password=secret;Encrypt=True;Trust Server Certificate=False;");
|
||||
|
||||
Assert.Contains("Data Source=localhost", display, StringComparison.Ordinal);
|
||||
Assert.Contains("Initial Catalog=ZB", display, StringComparison.Ordinal);
|
||||
Assert.Contains("Encrypt=True", display, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("User", display, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("Password", display, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("secret", display, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("mxuser", display, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Authentication.Cookies;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
public sealed class DashboardCookieOptionsTests
|
||||
{
|
||||
/// <summary>Verifies that the application configures secure dashboard authentication cookies.</summary>
|
||||
[Fact]
|
||||
public async Task Build_ConfiguresSecureDashboardCookie()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
IOptionsMonitor<CookieAuthenticationOptions> optionsMonitor = app.Services
|
||||
.GetRequiredService<IOptionsMonitor<CookieAuthenticationOptions>>();
|
||||
|
||||
CookieAuthenticationOptions options = optionsMonitor.Get(
|
||||
DashboardAuthenticationDefaults.AuthenticationScheme);
|
||||
|
||||
Assert.Equal(DashboardAuthenticationDefaults.CookieName, options.Cookie.Name);
|
||||
Assert.True(options.Cookie.HttpOnly);
|
||||
Assert.Equal(CookieSecurePolicy.Always, options.Cookie.SecurePolicy);
|
||||
Assert.Equal(SameSiteMode.Strict, options.Cookie.SameSite);
|
||||
Assert.Equal("/", options.Cookie.Path);
|
||||
Assert.Equal("/dashboard/login", options.LoginPath);
|
||||
Assert.Equal("/dashboard/logout", options.LogoutPath);
|
||||
Assert.Equal("/dashboard/denied", options.AccessDeniedPath);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,613 @@
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
using ZB.MOM.WW.MxGateway.Server.Galaxy;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
public sealed class DashboardSnapshotServiceTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Verifies snapshot returns empty collections and healthy status when registry is empty.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetSnapshot_WhenRegistryEmpty_ReturnsEmptyOperationalState()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics);
|
||||
|
||||
DashboardSnapshot snapshot = service.GetSnapshot();
|
||||
|
||||
Assert.Empty(snapshot.Sessions);
|
||||
Assert.Empty(snapshot.Workers);
|
||||
Assert.Empty(snapshot.Faults);
|
||||
Assert.Contains(snapshot.Metrics, metric => metric.Name == "mxgateway.sessions.open" && metric.Value == 0);
|
||||
Assert.Equal("Healthy", snapshot.GatewayStatus);
|
||||
Assert.NotNull(snapshot.Configuration);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies snapshot projects active, faulted, and closed session states with worker and metrics data.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetSnapshot_ProjectsActiveAndFaultedSessionsWorkersMetricsAndFaults()
|
||||
{
|
||||
SessionRegistry registry = new();
|
||||
GatewaySession activeSession = CreateSession(
|
||||
"session-active",
|
||||
"client-one",
|
||||
DateTimeOffset.Parse("2026-04-26T10:00:00Z", CultureInfo.InvariantCulture));
|
||||
activeSession.AttachWorkerClient(new FakeWorkerClient("session-active", 1201, WorkerClientState.Ready));
|
||||
activeSession.MarkReady();
|
||||
GatewaySession faultedSession = CreateSession(
|
||||
"session-faulted",
|
||||
"client-two",
|
||||
DateTimeOffset.Parse("2026-04-26T10:01:00Z", CultureInfo.InvariantCulture));
|
||||
faultedSession.AttachWorkerClient(new FakeWorkerClient("session-faulted", 1202, WorkerClientState.Faulted));
|
||||
faultedSession.MarkFaulted("worker pipe disconnected");
|
||||
GatewaySession closedSession = CreateSession(
|
||||
"session-closed",
|
||||
"client-three",
|
||||
DateTimeOffset.Parse("2026-04-26T09:59:00Z", CultureInfo.InvariantCulture));
|
||||
closedSession.AttachWorkerClient(new FakeWorkerClient("session-closed", 1203, WorkerClientState.Closed));
|
||||
closedSession.TransitionTo(SessionState.Closed);
|
||||
registry.TryAdd(activeSession);
|
||||
registry.TryAdd(faultedSession);
|
||||
registry.TryAdd(closedSession);
|
||||
using GatewayMetrics metrics = new();
|
||||
metrics.SessionOpened();
|
||||
metrics.SessionOpened();
|
||||
metrics.CommandStarted("Register");
|
||||
metrics.CommandFailed("Register", "WorkerFaulted", TimeSpan.FromMilliseconds(7));
|
||||
metrics.EventReceived("session-active", "OnDataChange");
|
||||
metrics.Fault("WorkerFaulted");
|
||||
DashboardSnapshotService service = CreateService(registry, metrics);
|
||||
|
||||
DashboardSnapshot snapshot = service.GetSnapshot();
|
||||
|
||||
Assert.Equal(3, snapshot.Sessions.Count);
|
||||
Assert.Equal("session-faulted", snapshot.Sessions[0].SessionId);
|
||||
Assert.Equal(SessionState.Faulted, snapshot.Sessions[0].State);
|
||||
DashboardSessionSummary activeSummary = Assert.Single(
|
||||
snapshot.Sessions,
|
||||
session => session.SessionId == "session-active");
|
||||
Assert.Equal(1, activeSummary.EventsReceived);
|
||||
Assert.Equal(2, snapshot.Workers.Count);
|
||||
Assert.DoesNotContain(snapshot.Workers, worker => worker.SessionId == "session-closed");
|
||||
Assert.Contains(snapshot.Metrics, metric => metric.Name == "mxgateway.commands.started" && metric.Value == 1);
|
||||
Assert.Contains(
|
||||
snapshot.Metrics,
|
||||
metric => metric.Name == "mxgateway.events.received"
|
||||
&& metric.Dimension == "OnDataChange"
|
||||
&& metric.Value == 1);
|
||||
DashboardFaultSummary fault = Assert.Single(snapshot.Faults);
|
||||
Assert.Equal("Worker", fault.Source);
|
||||
Assert.Equal("session-faulted", fault.SessionId);
|
||||
Assert.Equal("worker pipe disconnected", fault.Message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies snapshot redacts sensitive values from client identity, session name, and fault messages.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetSnapshot_RedactsSecretsFromSessionAndFaultFields()
|
||||
{
|
||||
SessionRegistry registry = new();
|
||||
GatewaySession session = CreateSession(
|
||||
"session-redacted",
|
||||
"Bearer mxgw_admin_super-secret",
|
||||
DateTimeOffset.Parse("2026-04-26T10:00:00Z", CultureInfo.InvariantCulture),
|
||||
clientSessionName: "password=hunter2",
|
||||
clientCorrelationId: "token=abc123");
|
||||
session.MarkFaulted("secret=credential-value");
|
||||
registry.TryAdd(session);
|
||||
using GatewayMetrics metrics = new();
|
||||
DashboardSnapshotService service = CreateService(registry, metrics);
|
||||
|
||||
DashboardSnapshot snapshot = service.GetSnapshot();
|
||||
|
||||
DashboardSessionSummary summary = Assert.Single(snapshot.Sessions);
|
||||
Assert.Equal("Bearer mxgw_admin_[redacted]", summary.ClientIdentity);
|
||||
Assert.Equal("[redacted]", summary.ClientSessionName);
|
||||
Assert.Equal("[redacted]", summary.ClientCorrelationId);
|
||||
Assert.Equal("[redacted]", summary.LastFault);
|
||||
Assert.Equal("[redacted]", Assert.Single(snapshot.Faults).Message);
|
||||
Assert.Equal("[redacted]", snapshot.Configuration.Authentication.PepperSecretName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies snapshot generation does not mutate session or worker client state.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetSnapshot_DoesNotMutateSessionOrWorkerState()
|
||||
{
|
||||
SessionRegistry registry = new();
|
||||
GatewaySession session = CreateSession(
|
||||
"session-active",
|
||||
"client-one",
|
||||
DateTimeOffset.Parse("2026-04-26T10:00:00Z", CultureInfo.InvariantCulture));
|
||||
FakeWorkerClient workerClient = new("session-active", 1201, WorkerClientState.Ready);
|
||||
session.AttachWorkerClient(workerClient);
|
||||
session.MarkReady();
|
||||
registry.TryAdd(session);
|
||||
using GatewayMetrics metrics = new();
|
||||
DashboardSnapshotService service = CreateService(registry, metrics);
|
||||
|
||||
service.GetSnapshot();
|
||||
service.GetSnapshot();
|
||||
|
||||
Assert.Equal(1, registry.ActiveCount);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
Assert.Equal(WorkerClientState.Ready, workerClient.State);
|
||||
Assert.Equal(0, workerClient.StartCount);
|
||||
Assert.Equal(0, workerClient.ShutdownCount);
|
||||
Assert.Equal(0, workerClient.KillCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies snapshot respects configured limits for recent sessions and faults.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetSnapshot_AppliesRecentSessionAndFaultLimits()
|
||||
{
|
||||
SessionRegistry registry = new();
|
||||
GatewaySession olderSession = CreateSession(
|
||||
"session-older",
|
||||
"client-one",
|
||||
DateTimeOffset.Parse("2026-04-26T10:00:00Z", CultureInfo.InvariantCulture));
|
||||
GatewaySession newerSession = CreateSession(
|
||||
"session-newer",
|
||||
"client-two",
|
||||
DateTimeOffset.Parse("2026-04-26T10:01:00Z", CultureInfo.InvariantCulture));
|
||||
olderSession.MarkFaulted("older fault");
|
||||
newerSession.MarkFaulted("newer fault");
|
||||
registry.TryAdd(olderSession);
|
||||
registry.TryAdd(newerSession);
|
||||
using GatewayMetrics metrics = new();
|
||||
DashboardSnapshotService service = CreateService(
|
||||
registry,
|
||||
metrics,
|
||||
new GatewayOptions
|
||||
{
|
||||
Dashboard = new DashboardOptions
|
||||
{
|
||||
SnapshotIntervalMilliseconds = 1,
|
||||
RecentSessionLimit = 1,
|
||||
RecentFaultLimit = 1,
|
||||
},
|
||||
});
|
||||
|
||||
DashboardSnapshot snapshot = service.GetSnapshot();
|
||||
|
||||
Assert.Equal("session-newer", Assert.Single(snapshot.Sessions).SessionId);
|
||||
Assert.Equal("session-newer", Assert.Single(snapshot.Faults).SessionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies snapshot projects Galaxy hierarchy cache data including templates and categories.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetSnapshot_ProjectsGalaxySummaryFromHierarchyCache()
|
||||
{
|
||||
GalaxyHierarchyCacheEntry entry = GalaxyHierarchyCacheEntry.Empty with
|
||||
{
|
||||
Status = GalaxyCacheStatus.Healthy,
|
||||
Sequence = 7,
|
||||
LastQueriedAt = DateTimeOffset.Parse("2026-04-28T11:30:00Z", CultureInfo.InvariantCulture),
|
||||
LastSuccessAt = DateTimeOffset.Parse("2026-04-28T11:30:00Z", CultureInfo.InvariantCulture),
|
||||
LastDeployTime = DateTimeOffset.Parse("2026-04-28T09:00:00Z", CultureInfo.InvariantCulture),
|
||||
DashboardSummary = new DashboardGalaxySummary(
|
||||
DashboardGalaxyStatus.Healthy,
|
||||
LastQueriedAt: DateTimeOffset.Parse("2026-04-28T11:30:00Z", CultureInfo.InvariantCulture),
|
||||
LastSuccessAt: DateTimeOffset.Parse("2026-04-28T11:30:00Z", CultureInfo.InvariantCulture),
|
||||
LastDeployTime: DateTimeOffset.Parse("2026-04-28T09:00:00Z", CultureInfo.InvariantCulture),
|
||||
LastError: null,
|
||||
ObjectCount: 3,
|
||||
AreaCount: 1,
|
||||
AttributeCount: 2,
|
||||
HistorizedAttributeCount: 1,
|
||||
AlarmAttributeCount: 1,
|
||||
TopTemplates:
|
||||
[
|
||||
new DashboardGalaxyTemplateUsage("$Pump", 2),
|
||||
new DashboardGalaxyTemplateUsage("$Area", 1),
|
||||
],
|
||||
ObjectCategories:
|
||||
[
|
||||
new DashboardGalaxyCategoryCount(10, "UserDefined", 2),
|
||||
new DashboardGalaxyCategoryCount(13, "Area", 1),
|
||||
]),
|
||||
ObjectCount = 3,
|
||||
AreaCount = 1,
|
||||
AttributeCount = 2,
|
||||
HistorizedAttributeCount = 1,
|
||||
AlarmAttributeCount = 1,
|
||||
};
|
||||
using GatewayMetrics metrics = new();
|
||||
DashboardSnapshotService service = CreateService(
|
||||
new SessionRegistry(),
|
||||
metrics,
|
||||
galaxyHierarchyCache: new StubGalaxyHierarchyCache(entry));
|
||||
|
||||
DashboardSnapshot snapshot = service.GetSnapshot();
|
||||
|
||||
Assert.Equal(DashboardGalaxyStatus.Healthy, snapshot.Galaxy.Status);
|
||||
Assert.Equal(3, snapshot.Galaxy.ObjectCount);
|
||||
Assert.Equal(1, snapshot.Galaxy.AreaCount);
|
||||
Assert.Equal(2, snapshot.Galaxy.AttributeCount);
|
||||
Assert.Equal("$Pump", Assert.Single(snapshot.Galaxy.TopTemplates, t => t.TemplateName == "$Pump").TemplateName);
|
||||
Assert.Equal(2, snapshot.Galaxy.TopTemplates.First(t => t.TemplateName == "$Pump").InstanceCount);
|
||||
Assert.Contains(snapshot.Galaxy.ObjectCategories, c => c.CategoryName == "UserDefined" && c.ObjectCount == 2);
|
||||
Assert.Contains(snapshot.Galaxy.ObjectCategories, c => c.CategoryName == "Area" && c.ObjectCount == 1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies snapshot watcher cancels cleanly when subscriber cancels.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetSnapshot_DoesNotSynchronouslyListApiKeys()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
CountingApiKeyAdminStore apiKeyAdminStore = new();
|
||||
DashboardSnapshotService service = CreateService(
|
||||
new SessionRegistry(),
|
||||
metrics,
|
||||
apiKeyAdminStore: apiKeyAdminStore);
|
||||
|
||||
DashboardSnapshot snapshot = service.GetSnapshot();
|
||||
|
||||
Assert.Empty(snapshot.ApiKeys);
|
||||
Assert.Equal(0, apiKeyAdminStore.ListCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WatchSnapshotsAsync_RefreshesApiKeySummariesBeforeSnapshot()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
CountingApiKeyAdminStore apiKeyAdminStore = new(
|
||||
new ApiKeyRecord(
|
||||
KeyId: "operator01",
|
||||
KeyPrefix: "mxgw_operator01",
|
||||
SecretHash: [1, 2, 3],
|
||||
DisplayName: "Operator",
|
||||
Scopes: new HashSet<string>([GatewayScopes.MetadataRead], StringComparer.Ordinal),
|
||||
Constraints: ApiKeyConstraints.Empty with
|
||||
{
|
||||
BrowseSubtrees = ["Area1/*"],
|
||||
},
|
||||
CreatedUtc: DateTimeOffset.Parse("2026-04-28T12:00:00Z", CultureInfo.InvariantCulture),
|
||||
LastUsedUtc: null,
|
||||
RevokedUtc: null));
|
||||
DashboardSnapshotService service = CreateService(
|
||||
new SessionRegistry(),
|
||||
metrics,
|
||||
apiKeyAdminStore: apiKeyAdminStore);
|
||||
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(2));
|
||||
await using IAsyncEnumerator<DashboardSnapshot> enumerator = service
|
||||
.WatchSnapshotsAsync(cancellation.Token)
|
||||
.GetAsyncEnumerator(cancellation.Token);
|
||||
|
||||
Assert.True(await enumerator.MoveNextAsync());
|
||||
DashboardSnapshot snapshot = enumerator.Current;
|
||||
|
||||
DashboardApiKeySummary key = Assert.Single(snapshot.ApiKeys);
|
||||
Assert.Equal("operator01", key.KeyId);
|
||||
Assert.Equal(["Area1/*"], key.Constraints.BrowseSubtrees);
|
||||
Assert.Equal(1, apiKeyAdminStore.ListCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WatchSnapshotsAsync_WhenApiKeyRefreshFails_ReusesPreviousSummaries()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
SequencedApiKeyAdminStore apiKeyAdminStore = new(
|
||||
new ApiKeyRecord(
|
||||
KeyId: "operator01",
|
||||
KeyPrefix: "mxgw_operator01",
|
||||
SecretHash: [1, 2, 3],
|
||||
DisplayName: "Operator",
|
||||
Scopes: new HashSet<string>([GatewayScopes.MetadataRead], StringComparer.Ordinal),
|
||||
Constraints: ApiKeyConstraints.Empty,
|
||||
CreatedUtc: DateTimeOffset.Parse("2026-04-28T12:00:00Z", CultureInfo.InvariantCulture),
|
||||
LastUsedUtc: null,
|
||||
RevokedUtc: null));
|
||||
DashboardSnapshotService service = CreateService(
|
||||
new SessionRegistry(),
|
||||
metrics,
|
||||
new GatewayOptions
|
||||
{
|
||||
Dashboard = new DashboardOptions
|
||||
{
|
||||
SnapshotIntervalMilliseconds = 1,
|
||||
},
|
||||
},
|
||||
apiKeyAdminStore: apiKeyAdminStore);
|
||||
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(2));
|
||||
await using IAsyncEnumerator<DashboardSnapshot> enumerator = service
|
||||
.WatchSnapshotsAsync(cancellation.Token)
|
||||
.GetAsyncEnumerator(cancellation.Token);
|
||||
|
||||
Assert.True(await enumerator.MoveNextAsync());
|
||||
DashboardSnapshot first = enumerator.Current;
|
||||
apiKeyAdminStore.FailNext = true;
|
||||
|
||||
Assert.True(await enumerator.MoveNextAsync());
|
||||
DashboardSnapshot second = enumerator.Current;
|
||||
|
||||
Assert.Equal("operator01", Assert.Single(first.ApiKeys).KeyId);
|
||||
Assert.Equal("operator01", Assert.Single(second.ApiKeys).KeyId);
|
||||
Assert.Equal(2, apiKeyAdminStore.ListCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WatchSnapshotsAsync_WhenSubscriberCancels_DisposesCleanly()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
DashboardSnapshotService service = CreateService(
|
||||
new SessionRegistry(),
|
||||
metrics,
|
||||
new GatewayOptions
|
||||
{
|
||||
Dashboard = new DashboardOptions
|
||||
{
|
||||
SnapshotIntervalMilliseconds = 1,
|
||||
},
|
||||
});
|
||||
using CancellationTokenSource cancellation = new();
|
||||
await using IAsyncEnumerator<DashboardSnapshot> enumerator = service
|
||||
.WatchSnapshotsAsync(cancellation.Token)
|
||||
.GetAsyncEnumerator();
|
||||
|
||||
Assert.True(await enumerator.MoveNextAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(1)));
|
||||
await cancellation.CancelAsync();
|
||||
bool hasNext = await enumerator.MoveNextAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(1));
|
||||
|
||||
Assert.False(hasNext);
|
||||
}
|
||||
|
||||
private static DashboardSnapshotService CreateService(
|
||||
SessionRegistry registry,
|
||||
GatewayMetrics metrics,
|
||||
GatewayOptions? options = null,
|
||||
IGalaxyHierarchyCache? galaxyHierarchyCache = null,
|
||||
IApiKeyAdminStore? apiKeyAdminStore = null)
|
||||
{
|
||||
GatewayOptions resolvedOptions = options ?? new GatewayOptions
|
||||
{
|
||||
Dashboard = new DashboardOptions
|
||||
{
|
||||
SnapshotIntervalMilliseconds = 1,
|
||||
},
|
||||
};
|
||||
GatewayConfigurationProvider configurationProvider = new(Options.Create(resolvedOptions));
|
||||
|
||||
return new DashboardSnapshotService(
|
||||
registry,
|
||||
metrics,
|
||||
configurationProvider,
|
||||
galaxyHierarchyCache ?? new StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry.Empty),
|
||||
apiKeyAdminStore ?? new FakeApiKeyAdminStore(),
|
||||
Options.Create(resolvedOptions));
|
||||
}
|
||||
|
||||
private sealed class StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry current) : IGalaxyHierarchyCache
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the current Galaxy hierarchy cache entry.
|
||||
/// </summary>
|
||||
public GalaxyHierarchyCacheEntry Current { get; } = current;
|
||||
|
||||
/// <summary>
|
||||
/// Refreshes the cache asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Completed task.</returns>
|
||||
public Task RefreshAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <summary>
|
||||
/// Waits for the first cache load asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Completed task.</returns>
|
||||
public Task WaitForFirstLoadAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
private class FakeApiKeyAdminStore : IApiKeyAdminStore
|
||||
{
|
||||
public Task CreateAsync(ApiKeyCreateRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public virtual Task<IReadOnlyList<ApiKeyRecord>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<ApiKeyRecord>>([]);
|
||||
}
|
||||
|
||||
public Task<bool> RevokeAsync(
|
||||
string keyId,
|
||||
DateTimeOffset revokedUtc,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
public Task<bool> RotateAsync(
|
||||
string keyId,
|
||||
byte[] secretHash,
|
||||
DateTimeOffset rotatedUtc,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
}
|
||||
|
||||
private class CountingApiKeyAdminStore(params ApiKeyRecord[] records) : FakeApiKeyAdminStore
|
||||
{
|
||||
public int ListCount { get; protected set; }
|
||||
|
||||
public override Task<IReadOnlyList<ApiKeyRecord>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
ListCount++;
|
||||
return Task.FromResult<IReadOnlyList<ApiKeyRecord>>(records);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SequencedApiKeyAdminStore(ApiKeyRecord record) : CountingApiKeyAdminStore(record)
|
||||
{
|
||||
public bool FailNext { get; set; }
|
||||
|
||||
public override Task<IReadOnlyList<ApiKeyRecord>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (FailNext)
|
||||
{
|
||||
FailNext = false;
|
||||
ListCount++;
|
||||
throw new InvalidOperationException("Simulated SQLite failure.");
|
||||
}
|
||||
|
||||
return base.ListAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private static GatewaySession CreateSession(
|
||||
string sessionId,
|
||||
string? clientIdentity,
|
||||
DateTimeOffset openedAt,
|
||||
string? clientSessionName = "test-session",
|
||||
string? clientCorrelationId = "client-correlation")
|
||||
{
|
||||
return new GatewaySession(
|
||||
sessionId,
|
||||
"mxaccess",
|
||||
$"mxaccess-gateway-1-{sessionId}",
|
||||
"nonce",
|
||||
clientIdentity,
|
||||
clientSessionName,
|
||||
clientCorrelationId,
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromSeconds(5),
|
||||
TimeSpan.FromSeconds(5),
|
||||
openedAt);
|
||||
}
|
||||
|
||||
private sealed class FakeWorkerClient(
|
||||
string sessionId,
|
||||
int? processId,
|
||||
WorkerClientState state) : IWorkerClient
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the session identifier.
|
||||
/// </summary>
|
||||
public string SessionId { get; } = sessionId;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the process identifier.
|
||||
/// </summary>
|
||||
public int? ProcessId { get; } = processId;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current worker client state.
|
||||
/// </summary>
|
||||
public WorkerClientState State { get; private set; } = state;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the timestamp of the last heartbeat.
|
||||
/// </summary>
|
||||
public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.Parse("2026-04-26T10:02:00Z", CultureInfo.InvariantCulture);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the count of start invocations.
|
||||
/// </summary>
|
||||
public int StartCount { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the count of shutdown invocations.
|
||||
/// </summary>
|
||||
public int ShutdownCount { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the count of kill invocations.
|
||||
/// </summary>
|
||||
public int KillCount { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Starts the worker client asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Completed task.</returns>
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
StartCount++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invokes a worker command asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="command">The command to invoke.</param>
|
||||
/// <param name="timeout">Command timeout.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Command reply.</returns>
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
WorkerCommand command,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new WorkerCommandReply());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads events from the worker asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Async enumerable of worker events.</returns>
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Shuts down the worker client asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="timeout">Shutdown timeout.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Completed task.</returns>
|
||||
public Task ShutdownAsync(
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ShutdownCount++;
|
||||
State = WorkerClientState.Closed;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Terminates the worker client.
|
||||
/// </summary>
|
||||
/// <param name="reason">Reason for termination.</param>
|
||||
public void Kill(string reason)
|
||||
{
|
||||
KillCount++;
|
||||
State = WorkerClientState.Faulted;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases resources used by this worker client.
|
||||
/// </summary>
|
||||
/// <returns>Completed value task.</returns>
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway;
|
||||
|
||||
public sealed class GatewayApplicationTests
|
||||
{
|
||||
/// <summary>Verifies that Build maps the live health check endpoint.</summary>
|
||||
[Fact]
|
||||
public async Task Build_MapsLiveHealthEndpoint()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
|
||||
RouteEndpoint endpoint = Assert.Single(
|
||||
((IEndpointRouteBuilder)app).DataSources
|
||||
.SelectMany(dataSource => dataSource.Endpoints)
|
||||
.OfType<RouteEndpoint>(),
|
||||
candidate => candidate.RoutePattern.RawText == "/health/live");
|
||||
|
||||
Assert.Equal("LiveHealth", endpoint.Metadata.GetMetadata<IEndpointNameMetadata>()?.EndpointName);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that Build registers the gateway metrics service.</summary>
|
||||
[Fact]
|
||||
public async Task Build_RegistersGatewayMetrics()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
|
||||
GatewayMetrics metrics = app.Services.GetRequiredService<GatewayMetrics>();
|
||||
|
||||
Assert.NotNull(metrics);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that Build maps dashboard and authentication endpoints when the dashboard is enabled.</summary>
|
||||
[Fact]
|
||||
public async Task Build_WhenDashboardEnabled_MapsBlazorDashboardAndAuthEndpoints()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
IReadOnlyList<RouteEndpoint> endpoints = GetRouteEndpoints(app);
|
||||
|
||||
Assert.Contains(endpoints, endpoint => endpoint.RoutePattern.RawText == "/dashboard/");
|
||||
Assert.Contains(endpoints, endpoint => endpoint.RoutePattern.RawText == "/dashboard/sessions");
|
||||
Assert.Contains(endpoints, endpoint => endpoint.RoutePattern.RawText == "/dashboard/workers");
|
||||
Assert.Contains(endpoints, endpoint => endpoint.RoutePattern.RawText == "/dashboard/events");
|
||||
Assert.Contains(endpoints, endpoint => endpoint.RoutePattern.RawText == "/dashboard/settings");
|
||||
Assert.Contains(endpoints, endpoint =>
|
||||
endpoint.Metadata.GetMetadata<IEndpointNameMetadata>()?.EndpointName == "DashboardLogin");
|
||||
Assert.Contains(endpoints, endpoint =>
|
||||
endpoint.Metadata.GetMetadata<IEndpointNameMetadata>()?.EndpointName == "DashboardLogout");
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the dashboard login, logout, and denied endpoints allow anonymous access.</summary>
|
||||
[Fact]
|
||||
public async Task Build_WhenDashboardEnabled_AuthEndpointsAllowAnonymousAccess()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
IReadOnlyList<RouteEndpoint> endpoints = GetRouteEndpoints(app);
|
||||
|
||||
string[] anonymousEndpointNames =
|
||||
["DashboardLogin", "DashboardLoginPost", "DashboardLogout", "DashboardAccessDenied"];
|
||||
foreach (string endpointName in anonymousEndpointNames)
|
||||
{
|
||||
RouteEndpoint endpoint = Assert.Single(
|
||||
endpoints,
|
||||
candidate => candidate.Metadata.GetMetadata<IEndpointNameMetadata>()?.EndpointName == endpointName);
|
||||
|
||||
Assert.NotNull(endpoint.Metadata.GetMetadata<IAllowAnonymous>());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Verifies that dashboard Razor component routes require the dashboard authorization policy.</summary>
|
||||
[Fact]
|
||||
public async Task Build_WhenDashboardEnabled_ComponentRoutesRequireAuthorization()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
IReadOnlyList<RouteEndpoint> endpoints = GetRouteEndpoints(app);
|
||||
|
||||
string[] componentRoutes =
|
||||
["/dashboard/", "/dashboard/sessions", "/dashboard/workers", "/dashboard/events", "/dashboard/settings"];
|
||||
foreach (string route in componentRoutes)
|
||||
{
|
||||
RouteEndpoint[] matches = endpoints
|
||||
.Where(endpoint => endpoint.RoutePattern.RawText == route)
|
||||
.ToArray();
|
||||
|
||||
Assert.NotEmpty(matches);
|
||||
Assert.All(matches, endpoint =>
|
||||
{
|
||||
IAuthorizeData? authorize = endpoint.Metadata.GetMetadata<IAuthorizeData>();
|
||||
Assert.NotNull(authorize);
|
||||
Assert.Equal(DashboardAuthenticationDefaults.AuthorizationPolicy, authorize.Policy);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Server-020 reversal regression guard. The original Server-020 finding
|
||||
/// incorrectly concluded that the duplicate <c>@page "/dashboard/X"</c>
|
||||
/// directives were redundant because <c>MapGroup("/dashboard")</c>
|
||||
/// would prepend the prefix to all dashboard Razor pages. In practice
|
||||
/// Blazor SSR's <c>RouteTableFactory</c> matches against the raw
|
||||
/// <c>@page</c> template values (not against the endpoint-route
|
||||
/// prefix), so removing <c>@page "/dashboard/X"</c> left the dashboard
|
||||
/// unreachable at runtime (every page returned HTTP 500 with "Unable
|
||||
/// to find the provided template '/dashboard/'"). The duplicate
|
||||
/// <c>@page</c> directives are restored, and as a side effect the
|
||||
/// endpoint route table DOES carry the doubled <c>/dashboard/dashboard/X</c>
|
||||
/// shape (because <c>MapGroup("/dashboard")</c> prefixes the already-prefixed
|
||||
/// <c>@page "/dashboard/X"</c>). Those doubled endpoints are harmless —
|
||||
/// no client requests <c>/dashboard/dashboard/X</c> — and removing them
|
||||
/// requires either dropping <c>MapGroup</c> or the <c>@page</c>
|
||||
/// prefix. This test asserts only the positive contract: every
|
||||
/// dashboard page IS reachable under the canonical <c>/dashboard/X</c>
|
||||
/// route, which is what the Blazor router actually serves.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Build_WhenDashboardEnabled_RegistersCanonicalDashboardRoutes()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
IReadOnlyList<RouteEndpoint> endpoints = GetRouteEndpoints(app);
|
||||
|
||||
string[] canonicalRoutes =
|
||||
[
|
||||
"/dashboard/",
|
||||
"/dashboard/sessions",
|
||||
"/dashboard/workers",
|
||||
"/dashboard/events",
|
||||
"/dashboard/settings",
|
||||
"/dashboard/galaxy",
|
||||
"/dashboard/apikeys",
|
||||
"/dashboard/sessions/{SessionId}",
|
||||
];
|
||||
foreach (string canonical in canonicalRoutes)
|
||||
{
|
||||
Assert.Contains(endpoints, endpoint => endpoint.RoutePattern.RawText == canonical);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Build_WhenDashboardDisabled_DoesNotMapDashboardRoutes()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build(["--MxGateway:Dashboard:Enabled=false"]);
|
||||
IReadOnlyList<RouteEndpoint> endpoints = GetRouteEndpoints(app);
|
||||
|
||||
Assert.DoesNotContain(endpoints, endpoint =>
|
||||
endpoint.RoutePattern.RawText?.StartsWith("/dashboard", StringComparison.Ordinal) == true);
|
||||
Assert.DoesNotContain(endpoints, endpoint =>
|
||||
endpoint.Metadata.GetMetadata<IEndpointNameMetadata>()?.EndpointName?.StartsWith(
|
||||
"Dashboard",
|
||||
StringComparison.Ordinal) == true);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that StartAsync fails when gateway configuration is invalid.</summary>
|
||||
/// <param name="key">Configuration key to override.</param>
|
||||
/// <param name="value">Invalid configuration value.</param>
|
||||
/// <param name="expectedFailure">Expected validation error message.</param>
|
||||
[Theory]
|
||||
[InlineData(
|
||||
"MxGateway:Worker:ExecutablePath",
|
||||
"worker.dll",
|
||||
"MxGateway:Worker:ExecutablePath must point to a .exe file.")]
|
||||
[InlineData(
|
||||
"MxGateway:Events:QueueCapacity",
|
||||
"0",
|
||||
"MxGateway:Events:QueueCapacity must be greater than zero.")]
|
||||
[InlineData(
|
||||
"MxGateway:Authentication:PepperSecretName",
|
||||
"",
|
||||
"MxGateway:Authentication:PepperSecretName is required")]
|
||||
[InlineData(
|
||||
"MxGateway:Dashboard:PathBase",
|
||||
"dashboard",
|
||||
"MxGateway:Dashboard:PathBase must start with '/'.")]
|
||||
[InlineData(
|
||||
"MxGateway:Ldap:RequiredGroup",
|
||||
"",
|
||||
"MxGateway:Ldap:RequiredGroup is required when LDAP login is enabled.")]
|
||||
[InlineData(
|
||||
"MxGateway:Ldap:AllowInsecureLdap",
|
||||
"false",
|
||||
"MxGateway:Ldap:AllowInsecureLdap must be true when UseTls is false.")]
|
||||
public async Task StartAsync_InvalidGatewayConfiguration_FailsStartup(
|
||||
string key,
|
||||
string value,
|
||||
string expectedFailure)
|
||||
{
|
||||
// Bind an ephemeral port (:0) — xUnit runs test collections in parallel, so any
|
||||
// WebApplication-building test must avoid a fixed port to prevent a bind collision.
|
||||
await using WebApplication app = GatewayApplication.Build(
|
||||
[$"--{key}={value}", "--urls=http://127.0.0.1:0"]);
|
||||
|
||||
OptionsValidationException exception = await Assert.ThrowsAsync<OptionsValidationException>(
|
||||
() => app.StartAsync());
|
||||
|
||||
Assert.Contains(
|
||||
exception.Failures,
|
||||
failure => failure.Contains(expectedFailure, StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
private static IReadOnlyList<RouteEndpoint> GetRouteEndpoints(WebApplication app)
|
||||
{
|
||||
return ((IEndpointRouteBuilder)app).DataSources
|
||||
.SelectMany(dataSource => dataSource.Endpoints)
|
||||
.OfType<RouteEndpoint>()
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway;
|
||||
|
||||
public sealed class GatewayEndToEndFakeWorkerSmokeTests
|
||||
{
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
private const int ServerHandle = 1001;
|
||||
private const int ItemHandle = 2002;
|
||||
|
||||
/// <summary>
|
||||
/// Verifies gateway session lifecycle with a scripted fake worker: open, command, event, close.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task GatewayService_WithFakeWorker_CompletesSessionCommandEventAndClosePath()
|
||||
{
|
||||
ScriptedFakeWorkerProcessLauncher launcher = new();
|
||||
await using GatewayServiceFixture fixture = new(launcher);
|
||||
|
||||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||||
new OpenSessionRequest
|
||||
{
|
||||
ClientSessionName = "fake-worker-e2e",
|
||||
ClientCorrelationId = "open-correlation",
|
||||
CommandTimeout = Duration.FromTimeSpan(TestTimeout),
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
RecordingServerStreamWriter<MxEvent> eventWriter = new();
|
||||
Task streamTask = fixture.Service.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = openReply.SessionId },
|
||||
eventWriter,
|
||||
new TestServerCallContext());
|
||||
|
||||
MxCommandReply registerReply = await fixture.Service.Invoke(
|
||||
CreateRegisterRequest(openReply.SessionId),
|
||||
new TestServerCallContext());
|
||||
MxCommandReply addItemReply = await fixture.Service.Invoke(
|
||||
CreateAddItemRequest(openReply.SessionId, registerReply.Register.ServerHandle),
|
||||
new TestServerCallContext());
|
||||
MxCommandReply adviseReply = await fixture.Service.Invoke(
|
||||
CreateAdviseRequest(openReply.SessionId, registerReply.Register.ServerHandle, addItemReply.AddItem.ItemHandle),
|
||||
new TestServerCallContext());
|
||||
|
||||
MxEvent dataChange = await eventWriter.WaitForFirstMessageAsync(TestTimeout);
|
||||
|
||||
CloseSessionReply closeReply = await fixture.Service.CloseSession(
|
||||
new CloseSessionRequest
|
||||
{
|
||||
SessionId = openReply.SessionId,
|
||||
ClientCorrelationId = "close-correlation",
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
await streamTask.WaitAsync(TestTimeout);
|
||||
await launcher.WorkerTask.WaitAsync(TestTimeout);
|
||||
|
||||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||||
Assert.Equal(GatewayContractInfo.DefaultBackendName, openReply.BackendName);
|
||||
Assert.Equal(ScriptedFakeWorkerProcessLauncher.ProcessId, openReply.WorkerProcessId);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
|
||||
Assert.Equal(ServerHandle, registerReply.Register.ServerHandle);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
|
||||
Assert.Equal(ItemHandle, addItemReply.AddItem.ItemHandle);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
|
||||
Assert.Equal(MxEventFamily.OnDataChange, dataChange.Family);
|
||||
Assert.Equal(openReply.SessionId, dataChange.SessionId);
|
||||
Assert.Equal(ServerHandle, dataChange.ServerHandle);
|
||||
Assert.Equal(ItemHandle, dataChange.ItemHandle);
|
||||
Assert.Equal("scripted-value", dataChange.Value.StringValue);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, closeReply.ProtocolStatus.Code);
|
||||
Assert.Equal(SessionState.Closed, closeReply.FinalState);
|
||||
Assert.True(launcher.Process.HasExited);
|
||||
// MarkExited(0) is reached only after the scripted worker observed a WorkerShutdown
|
||||
// envelope and emitted its WorkerShutdownAck — anything else (a kill, a fault) would
|
||||
// have produced a non-zero exit code, so this pins the shutdown-ack handshake.
|
||||
Assert.Equal(0, launcher.Process.ExitCode);
|
||||
Assert.Equal(
|
||||
[MxCommandKind.Register, MxCommandKind.AddItem, MxCommandKind.Advise],
|
||||
launcher.CommandKinds);
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateRegisterRequest(string sessionId)
|
||||
{
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = sessionId,
|
||||
ClientCorrelationId = "register-correlation",
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.Register,
|
||||
Register = new RegisterCommand { ClientName = "fake-worker-e2e-client" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateAddItemRequest(
|
||||
string sessionId,
|
||||
int serverHandle)
|
||||
{
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = sessionId,
|
||||
ClientCorrelationId = "add-item-correlation",
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AddItem,
|
||||
AddItem = new AddItemCommand
|
||||
{
|
||||
ServerHandle = serverHandle,
|
||||
ItemDefinition = "Galaxy.Tag.Value",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateAdviseRequest(
|
||||
string sessionId,
|
||||
int serverHandle,
|
||||
int itemHandle)
|
||||
{
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = sessionId,
|
||||
ClientCorrelationId = "advise-correlation",
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.Advise,
|
||||
Advise = new AdviseCommand
|
||||
{
|
||||
ServerHandle = serverHandle,
|
||||
ItemHandle = itemHandle,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class GatewayServiceFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly GatewayMetrics _metrics = new();
|
||||
private readonly SessionRegistry _registry = new();
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of <see cref="GatewayServiceFixture"/>.
|
||||
/// </summary>
|
||||
/// <param name="launcher">Worker process launcher for the fixture.</param>
|
||||
public GatewayServiceFixture(IWorkerProcessLauncher launcher)
|
||||
{
|
||||
IOptions<GatewayOptions> options = Options.Create(CreateOptions());
|
||||
SessionWorkerClientFactory workerClientFactory = new(
|
||||
launcher,
|
||||
options,
|
||||
_metrics,
|
||||
NullLoggerFactory.Instance);
|
||||
SessionManager sessionManager = new(
|
||||
_registry,
|
||||
workerClientFactory,
|
||||
options,
|
||||
_metrics,
|
||||
logger: NullLogger<SessionManager>.Instance);
|
||||
MxAccessGrpcMapper mapper = new();
|
||||
EventStreamService eventStreamService = new(
|
||||
sessionManager,
|
||||
options,
|
||||
mapper,
|
||||
_metrics,
|
||||
NullLogger<EventStreamService>.Instance);
|
||||
|
||||
Service = new MxAccessGatewayService(
|
||||
sessionManager,
|
||||
new GatewayRequestIdentityAccessor(),
|
||||
new AllowAllConstraintEnforcer(),
|
||||
new MxAccessGrpcRequestValidator(),
|
||||
mapper,
|
||||
eventStreamService,
|
||||
_metrics,
|
||||
NullLogger<MxAccessGatewayService>.Instance,
|
||||
new FakeGatewayAlarmService());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configured gateway service instance.
|
||||
/// </summary>
|
||||
public MxAccessGatewayService Service { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Disposes all active sessions and metrics.
|
||||
/// </summary>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
foreach (GatewaySession session in _registry.Snapshot())
|
||||
{
|
||||
await session.DisposeAsync();
|
||||
}
|
||||
|
||||
_metrics.Dispose();
|
||||
}
|
||||
|
||||
private static GatewayOptions CreateOptions()
|
||||
{
|
||||
return new GatewayOptions
|
||||
{
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
StartupTimeoutSeconds = 5,
|
||||
ShutdownTimeoutSeconds = 5,
|
||||
HeartbeatIntervalSeconds = 30,
|
||||
HeartbeatGraceSeconds = 30,
|
||||
MaxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes,
|
||||
},
|
||||
Sessions = new SessionOptions
|
||||
{
|
||||
DefaultCommandTimeoutSeconds = 5,
|
||||
MaxSessions = 4,
|
||||
},
|
||||
Events = new EventOptions
|
||||
{
|
||||
QueueCapacity = 16,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ScriptedFakeWorkerProcessLauncher : IWorkerProcessLauncher
|
||||
{
|
||||
public const int ProcessId = 4680;
|
||||
private readonly ConcurrentQueue<MxCommandKind> _commandKinds = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the fake worker process instance.
|
||||
/// </summary>
|
||||
public FakeWorkerProcess Process { get; } = new(ProcessId);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the collection of command kinds processed by the worker.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<MxCommandKind> CommandKinds => _commandKinds.ToArray();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the worker's asynchronous task.
|
||||
/// </summary>
|
||||
public Task WorkerTask { get; private set; } = Task.CompletedTask;
|
||||
|
||||
/// <summary>
|
||||
/// Launches a new worker process and returns a handle to manage it.
|
||||
/// </summary>
|
||||
/// <param name="request">Worker process launch request parameters.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Worker process handle.</returns>
|
||||
public Task<WorkerProcessHandle> LaunchAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
WorkerTask = RunWorkerAsync(request, cancellationToken);
|
||||
|
||||
return Task.FromResult(new WorkerProcessHandle(
|
||||
Process,
|
||||
new WorkerProcessCommandLine("fake-worker.exe", []),
|
||||
DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
private async Task RunWorkerAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using FakeWorkerHarness harness = await FakeWorkerHarness.ConnectToGatewayPipeAsync(
|
||||
request.SessionId,
|
||||
request.Nonce,
|
||||
request.PipeName,
|
||||
request.ProtocolVersion,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
await harness.CompleteStartupAsync(ProcessId, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
WorkerEnvelope envelope = await harness.ReadGatewayEnvelopeAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerCommand)
|
||||
{
|
||||
await ReplyToCommandAsync(harness, envelope, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerShutdown)
|
||||
{
|
||||
await harness.SendShutdownAckAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
Process.MarkExited(0);
|
||||
return;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException($"Unexpected gateway envelope {envelope.BodyCase}.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReplyToCommandAsync(
|
||||
FakeWorkerHarness harness,
|
||||
WorkerEnvelope commandEnvelope,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
MxCommand command = commandEnvelope.WorkerCommand.Command;
|
||||
_commandKinds.Enqueue(command.Kind);
|
||||
|
||||
await harness.ReplyToCommandAsync(
|
||||
commandEnvelope,
|
||||
configureReply: reply => ConfigureReply(reply, command.Kind),
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (command.Kind == MxCommandKind.Advise)
|
||||
{
|
||||
await harness.EmitEventAsync(
|
||||
MxEventFamily.OnDataChange,
|
||||
cancellationToken,
|
||||
mxEvent =>
|
||||
{
|
||||
mxEvent.ServerHandle = command.Advise.ServerHandle;
|
||||
mxEvent.ItemHandle = command.Advise.ItemHandle;
|
||||
mxEvent.Quality = 192;
|
||||
mxEvent.Value = new MxValue
|
||||
{
|
||||
DataType = MxDataType.String,
|
||||
StringValue = "scripted-value",
|
||||
};
|
||||
mxEvent.OnDataChange = new OnDataChangeEvent();
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static void ConfigureReply(
|
||||
MxCommandReply reply,
|
||||
MxCommandKind kind)
|
||||
{
|
||||
switch (kind)
|
||||
{
|
||||
case MxCommandKind.Register:
|
||||
reply.Register = new RegisterReply { ServerHandle = ServerHandle };
|
||||
break;
|
||||
case MxCommandKind.AddItem:
|
||||
reply.AddItem = new AddItemReply { ItemHandle = ItemHandle };
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeWorkerProcess(int processId) : IWorkerProcess
|
||||
{
|
||||
private readonly TaskCompletionSource _exited = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the process identifier.
|
||||
/// </summary>
|
||||
public int Id { get; } = processId;
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the process has exited.
|
||||
/// </summary>
|
||||
public bool HasExited { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the exit code of the process.
|
||||
/// </summary>
|
||||
public int? ExitCode { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Waits for the process to exit asynchronously. Completes only when <see cref="Kill"/>
|
||||
/// or <see cref="MarkExited"/> has been called, so callers that observe completion can
|
||||
/// trust that exit actually happened (e.g., via the worker shutdown-ack path).
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that completes when the process has actually exited.</returns>
|
||||
public ValueTask WaitForExitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return new ValueTask(_exited.Task.WaitAsync(cancellationToken));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Terminates the process.
|
||||
/// </summary>
|
||||
/// <param name="entireProcessTree">Whether to kill the entire process tree.</param>
|
||||
public void Kill(bool entireProcessTree)
|
||||
{
|
||||
MarkExited(-1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases resources used by this process.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks the process as exited with the specified exit code.
|
||||
/// </summary>
|
||||
/// <param name="exitCode">The process exit code.</param>
|
||||
public void MarkExited(int exitCode)
|
||||
{
|
||||
HasExited = true;
|
||||
ExitCode = exitCode;
|
||||
_exited.TrySetResult();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,523 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
|
||||
|
||||
public sealed class EventStreamServiceTests
|
||||
{
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Verifies that events from the worker stream maintain their original sequence order.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_YieldsEventsInWorkerOrder()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
FakeSessionManager sessionManager = new(session);
|
||||
using GatewayMetrics metrics = new();
|
||||
EventStreamService service = CreateService(sessionManager, metrics: metrics);
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 10, MxEventFamily.OnDataChange));
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 11, MxEventFamily.OnWriteComplete));
|
||||
workerClient.CompleteAfterConfiguredEvents = true;
|
||||
|
||||
List<MxEvent> events = await CollectEventsAsync(service, session.SessionId);
|
||||
|
||||
Assert.Equal([10UL, 11UL], events.Select(mxEvent => mxEvent.WorkerSequence).ToArray());
|
||||
Assert.Equal(MxEventFamily.OnDataChange, events[0].Family);
|
||||
Assert.Equal(MxEventFamily.OnWriteComplete, events[1].Family);
|
||||
Assert.Equal(1, metrics.GetSnapshot().StreamDisconnects);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a second event subscriber is rejected when one is already active.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenSecondSubscriberStarts_RejectsClearly()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
EventStreamService service = CreateService(new FakeSessionManager(session));
|
||||
using CancellationTokenSource firstSubscriberCancellation = new();
|
||||
await using IAsyncEnumerator<MxEvent> firstSubscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), firstSubscriberCancellation.Token)
|
||||
.GetAsyncEnumerator(firstSubscriberCancellation.Token);
|
||||
Task<bool> firstMoveTask = firstSubscriber.MoveNextAsync().AsTask();
|
||||
|
||||
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1);
|
||||
await using IAsyncEnumerator<MxEvent> secondSubscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
|
||||
.GetAsyncEnumerator();
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await secondSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.EventSubscriberAlreadyActive, exception.ErrorCode);
|
||||
await firstSubscriberCancellation.CancelAsync();
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await firstMoveTask.WaitAsync(TestTimeout));
|
||||
await firstSubscriber.DisposeAsync();
|
||||
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 0);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that canceling an event stream detaches the subscriber cleanly.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenCanceled_DetachesSubscriber()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
EventStreamService service = CreateService(new FakeSessionManager(session));
|
||||
using CancellationTokenSource cancellationTokenSource = new();
|
||||
await using IAsyncEnumerator<MxEvent> subscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), cancellationTokenSource.Token)
|
||||
.GetAsyncEnumerator(cancellationTokenSource.Token);
|
||||
Task<bool> moveTask = subscriber.MoveNextAsync().AsTask();
|
||||
|
||||
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1);
|
||||
await cancellationTokenSource.CancelAsync();
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await moveTask.WaitAsync(TestTimeout));
|
||||
await subscriber.DisposeAsync();
|
||||
|
||||
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 0);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that disposing an event stream with buffered events resets the queue depth metric.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenDisposedWithBufferedEvents_ResetsStreamQueueDepth()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
using GatewayMetrics metrics = new();
|
||||
EventStreamService service = CreateService(
|
||||
new FakeSessionManager(session),
|
||||
metrics,
|
||||
queueCapacity: 8);
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 2, MxEventFamily.OnDataChange));
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 3, MxEventFamily.OnDataChange));
|
||||
workerClient.CompleteAfterConfiguredEvents = true;
|
||||
await using IAsyncEnumerator<MxEvent> subscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
|
||||
.GetAsyncEnumerator();
|
||||
|
||||
Assert.True(await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth > 0);
|
||||
|
||||
await subscriber.DisposeAsync();
|
||||
|
||||
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that queue depth metrics correctly track concurrent event streams across multiple sessions.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WithConcurrentStreams_TracksAggregateQueueDepth()
|
||||
{
|
||||
FakeWorkerClient firstWorkerClient = new();
|
||||
FakeWorkerClient secondWorkerClient = new();
|
||||
GatewaySession firstSession = CreateReadySession(firstWorkerClient, "session-events-1");
|
||||
GatewaySession secondSession = CreateReadySession(secondWorkerClient, "session-events-2");
|
||||
using GatewayMetrics metrics = new();
|
||||
EventStreamService service = CreateService(
|
||||
new FakeSessionManager(firstSession, secondSession),
|
||||
metrics,
|
||||
queueCapacity: 8);
|
||||
for (ulong sequence = 1; sequence <= 3; sequence++)
|
||||
{
|
||||
firstWorkerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
|
||||
secondWorkerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
|
||||
}
|
||||
|
||||
firstWorkerClient.CompleteAfterConfiguredEvents = true;
|
||||
secondWorkerClient.CompleteAfterConfiguredEvents = true;
|
||||
await using IAsyncEnumerator<MxEvent> firstSubscriber = service
|
||||
.StreamEventsAsync(CreateRequest(firstSession.SessionId), CancellationToken.None)
|
||||
.GetAsyncEnumerator();
|
||||
await using IAsyncEnumerator<MxEvent> secondSubscriber = service
|
||||
.StreamEventsAsync(CreateRequest(secondSession.SessionId), CancellationToken.None)
|
||||
.GetAsyncEnumerator();
|
||||
|
||||
Assert.True(await firstSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
Assert.True(await secondSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 4);
|
||||
|
||||
await firstSubscriber.DisposeAsync();
|
||||
|
||||
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 2);
|
||||
|
||||
await secondSubscriber.DisposeAsync();
|
||||
|
||||
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that event queue overflow faults the session and reports the overflow metric.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenStreamQueueOverflows_FaultsSessionAndReportsOverflow()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
using GatewayMetrics metrics = new();
|
||||
EventStreamService service = CreateService(
|
||||
new FakeSessionManager(session),
|
||||
metrics,
|
||||
queueCapacity: 1);
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 2, MxEventFamily.OnDataChange));
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 3, MxEventFamily.OnDataChange));
|
||||
workerClient.CompleteAfterConfiguredEvents = true;
|
||||
await using IAsyncEnumerator<MxEvent> subscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
|
||||
.GetAsyncEnumerator();
|
||||
|
||||
Assert.True(await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
await WaitUntilAsync(() => session.State == SessionState.Faulted);
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, exception.ErrorCode);
|
||||
Assert.Equal(SessionState.Faulted, session.State);
|
||||
Assert.Equal(1, metrics.GetSnapshot().QueueOverflows);
|
||||
Assert.Equal(1, metrics.GetSnapshot().Faults);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the disconnect backpressure policy disconnects the subscriber without faulting the session.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenStreamQueueOverflowsWithDisconnectPolicy_LeavesSessionReady()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
using GatewayMetrics metrics = new();
|
||||
EventStreamService service = CreateService(
|
||||
new FakeSessionManager(session),
|
||||
metrics,
|
||||
queueCapacity: 1,
|
||||
backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber);
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 2, MxEventFamily.OnDataChange));
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 3, MxEventFamily.OnDataChange));
|
||||
workerClient.CompleteAfterConfiguredEvents = true;
|
||||
await using IAsyncEnumerator<MxEvent> subscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
|
||||
.GetAsyncEnumerator();
|
||||
|
||||
Assert.True(await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, exception.ErrorCode);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snapshot.QueueOverflows);
|
||||
Assert.Equal(0, snapshot.Faults);
|
||||
Assert.Equal(1, snapshot.StreamDisconnects);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the event stream does not synthesize OperationComplete events from write completions.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_DoesNotSynthesizeOperationComplete()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
EventStreamService service = CreateService(new FakeSessionManager(session));
|
||||
workerClient.Events.Add(CreateWorkerEvent(sequence: 10, MxEventFamily.OnWriteComplete));
|
||||
workerClient.CompleteAfterConfiguredEvents = true;
|
||||
|
||||
List<MxEvent> events = await CollectEventsAsync(service, session.SessionId);
|
||||
|
||||
MxEvent mxEvent = Assert.Single(events);
|
||||
Assert.Equal(MxEventFamily.OnWriteComplete, mxEvent.Family);
|
||||
Assert.DoesNotContain(events, candidate => candidate.Family == MxEventFamily.OperationComplete);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a terminal fault from the worker event stream propagates and faults the session.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_WhenWorkerEventStreamFaults_PropagatesTerminalFault()
|
||||
{
|
||||
FakeWorkerClient workerClient = new()
|
||||
{
|
||||
TerminalException = new WorkerClientException(
|
||||
WorkerClientErrorCode.WorkerFaulted,
|
||||
"worker terminal fault"),
|
||||
};
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
using GatewayMetrics metrics = new();
|
||||
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
|
||||
await using IAsyncEnumerator<MxEvent> subscriber = service
|
||||
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
|
||||
.GetAsyncEnumerator();
|
||||
|
||||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||||
async () => await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
|
||||
Assert.Equal(WorkerClientErrorCode.WorkerFaulted, exception.ErrorCode);
|
||||
Assert.Equal(SessionState.Faulted, session.State);
|
||||
Assert.Equal(1, metrics.GetSnapshot().Faults);
|
||||
}
|
||||
|
||||
private static EventStreamService CreateService(
|
||||
FakeSessionManager sessionManager,
|
||||
GatewayMetrics? metrics = null,
|
||||
int queueCapacity = 8,
|
||||
EventBackpressurePolicy backpressurePolicy = EventBackpressurePolicy.FailFast)
|
||||
{
|
||||
return new EventStreamService(
|
||||
sessionManager,
|
||||
Options.Create(new GatewayOptions
|
||||
{
|
||||
Events = new EventOptions
|
||||
{
|
||||
QueueCapacity = queueCapacity,
|
||||
BackpressurePolicy = backpressurePolicy,
|
||||
},
|
||||
}),
|
||||
new MxAccessGrpcMapper(),
|
||||
metrics ?? new GatewayMetrics(),
|
||||
NullLogger<EventStreamService>.Instance);
|
||||
}
|
||||
|
||||
private static async Task<List<MxEvent>> CollectEventsAsync(
|
||||
EventStreamService service,
|
||||
string sessionId)
|
||||
{
|
||||
List<MxEvent> events = [];
|
||||
await foreach (MxEvent mxEvent in service
|
||||
.StreamEventsAsync(CreateRequest(sessionId), CancellationToken.None)
|
||||
.WithCancellation(CancellationToken.None))
|
||||
{
|
||||
events.Add(mxEvent);
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
private static StreamEventsRequest CreateRequest(string sessionId)
|
||||
{
|
||||
return new StreamEventsRequest
|
||||
{
|
||||
SessionId = sessionId,
|
||||
};
|
||||
}
|
||||
|
||||
private static GatewaySession CreateReadySession(
|
||||
FakeWorkerClient workerClient,
|
||||
string sessionId = "session-events")
|
||||
{
|
||||
GatewaySession session = new(
|
||||
sessionId,
|
||||
GatewayContractInfo.DefaultBackendName,
|
||||
"pipe",
|
||||
"nonce",
|
||||
"client",
|
||||
"client-session",
|
||||
"client-correlation",
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromSeconds(10),
|
||||
DateTimeOffset.UtcNow);
|
||||
session.AttachWorkerClient(workerClient);
|
||||
session.MarkReady();
|
||||
|
||||
return session;
|
||||
}
|
||||
|
||||
private static WorkerEvent CreateWorkerEvent(
|
||||
ulong sequence,
|
||||
MxEventFamily family)
|
||||
{
|
||||
MxEvent mxEvent = new()
|
||||
{
|
||||
SessionId = "session-events",
|
||||
Family = family,
|
||||
WorkerSequence = sequence,
|
||||
};
|
||||
|
||||
switch (family)
|
||||
{
|
||||
case MxEventFamily.OnDataChange:
|
||||
mxEvent.OnDataChange = new OnDataChangeEvent();
|
||||
break;
|
||||
case MxEventFamily.OnWriteComplete:
|
||||
mxEvent.OnWriteComplete = new OnWriteCompleteEvent();
|
||||
break;
|
||||
case MxEventFamily.OperationComplete:
|
||||
mxEvent.OperationComplete = new OperationCompleteEvent();
|
||||
break;
|
||||
case MxEventFamily.OnBufferedDataChange:
|
||||
mxEvent.OnBufferedDataChange = new OnBufferedDataChangeEvent();
|
||||
break;
|
||||
}
|
||||
|
||||
return new WorkerEvent
|
||||
{
|
||||
Event = mxEvent,
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> predicate)
|
||||
{
|
||||
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
|
||||
while (!predicate())
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake session manager for testing event streams.</summary>
|
||||
private sealed class FakeSessionManager : ISessionManager
|
||||
{
|
||||
private readonly IReadOnlyDictionary<string, GatewaySession> _sessions;
|
||||
|
||||
/// <summary>Initializes a new instance of the FakeSessionManager.</summary>
|
||||
/// <param name="sessions">Sessions to manage.</param>
|
||||
public FakeSessionManager(params GatewaySession[] sessions)
|
||||
{
|
||||
_sessions = sessions.ToDictionary(session => session.SessionId, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<GatewaySession> OpenSessionAsync(
|
||||
SessionOpenRequest request,
|
||||
string? clientIdentity,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(_sessions.Values.First());
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool TryGetSession(
|
||||
string sessionId,
|
||||
out GatewaySession gatewaySession)
|
||||
{
|
||||
return _sessions.TryGetValue(sessionId, out gatewaySession!);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
string sessionId,
|
||||
WorkerCommand command,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new WorkerCommandReply());
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
string sessionId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return _sessions[sessionId].ReadEventsAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<SessionCloseResult> CloseSessionAsync(
|
||||
string sessionId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<int> CloseExpiredLeasesAsync(
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake worker client for testing event streams.</summary>
|
||||
private sealed class FakeWorkerClient : IWorkerClient
|
||||
{
|
||||
/// <summary>Gets the list of queued worker events.</summary>
|
||||
public List<WorkerEvent> Events { get; } = [];
|
||||
|
||||
/// <summary>Gets or sets whether to complete the event stream after configured events are yielded.</summary>
|
||||
public bool CompleteAfterConfiguredEvents { get; set; }
|
||||
|
||||
/// <summary>Gets or sets an optional exception to throw as a terminal event stream fault.</summary>
|
||||
public Exception? TerminalException { get; init; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public string SessionId { get; } = "session-events";
|
||||
|
||||
/// <inheritdoc />
|
||||
public int? ProcessId { get; } = 4321;
|
||||
|
||||
/// <inheritdoc />
|
||||
public WorkerClientState State { get; private set; } = WorkerClientState.Ready;
|
||||
|
||||
/// <inheritdoc />
|
||||
public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.UtcNow;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
WorkerCommand command,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new WorkerCommandReply());
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (WorkerEvent workerEvent in Events)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
yield return workerEvent;
|
||||
}
|
||||
|
||||
if (TerminalException is not null)
|
||||
{
|
||||
throw TerminalException;
|
||||
}
|
||||
|
||||
if (CompleteAfterConfiguredEvents)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ShutdownAsync(
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
State = WorkerClientState.Closed;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Kill(string reason)
|
||||
{
|
||||
State = WorkerClientState.Faulted;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
using ZB.MOM.WW.MxGateway.Server.Galaxy;
|
||||
using ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
|
||||
|
||||
public sealed class GalaxyRepositoryGrpcServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchy_ReturnsRequestedPageAndTotals()
|
||||
{
|
||||
GalaxyRepositoryGrpcService service = CreateService(CreateEntry(CreateObjects(3)));
|
||||
|
||||
DiscoverHierarchyReply reply = await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
PageSize = 2,
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(2, reply.Objects.Count);
|
||||
Assert.Equal("Object_001", reply.Objects[0].TagName);
|
||||
Assert.Equal("Object_002", reply.Objects[1].TagName);
|
||||
Assert.StartsWith("7:", reply.NextPageToken, StringComparison.Ordinal);
|
||||
Assert.EndsWith(":2", reply.NextPageToken, StringComparison.Ordinal);
|
||||
Assert.Equal(3, reply.TotalObjectCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchy_WithNextPageToken_ReturnsRemainingObjects()
|
||||
{
|
||||
GalaxyRepositoryGrpcService service = CreateService(CreateEntry(CreateObjects(3)));
|
||||
DiscoverHierarchyReply firstPage = await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
PageSize = 2,
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
DiscoverHierarchyReply reply = await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
PageSize = 2,
|
||||
PageToken = firstPage.NextPageToken,
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
GalaxyObject item = Assert.Single(reply.Objects);
|
||||
Assert.Equal("Object_003", item.TagName);
|
||||
Assert.Equal("", reply.NextPageToken);
|
||||
Assert.Equal(3, reply.TotalObjectCount);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("-1", 1)]
|
||||
[InlineData("not-an-offset", 1)]
|
||||
[InlineData("7:4", 1)]
|
||||
[InlineData("6:2", 1)]
|
||||
[InlineData("", -1)]
|
||||
public async Task DiscoverHierarchy_WithInvalidPagingArguments_ReturnsInvalidArgument(
|
||||
string pageToken,
|
||||
int pageSize)
|
||||
{
|
||||
GalaxyRepositoryGrpcService service = CreateService(CreateEntry(CreateObjects(3)));
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
PageSize = pageSize,
|
||||
PageToken = pageToken,
|
||||
},
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.InvalidArgument, exception.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchy_WithSubtreeRootAndDepth_FiltersDescendants()
|
||||
{
|
||||
GalaxyRepositoryGrpcService service = CreateService(CreateEntry(CreateFilterObjects()));
|
||||
|
||||
DiscoverHierarchyReply reply = await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
RootContainedPath = "Area1/Line3",
|
||||
MaxDepth = 1,
|
||||
PageSize = 10,
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(["Line3", "Pump_001", "Valve_001"], reply.Objects.Select(obj => obj.TagName));
|
||||
Assert.Equal(3, reply.TotalObjectCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchy_WithServerSideFilters_AppliesAllFiltersAndOmitsAttributes()
|
||||
{
|
||||
GalaxyRepositoryGrpcService service = CreateService(CreateEntry(CreateFilterObjects()));
|
||||
|
||||
DiscoverHierarchyReply reply = await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
RootTagName = "Area1",
|
||||
TagNameGlob = "Pump_*",
|
||||
AlarmBearingOnly = true,
|
||||
HistorizedOnly = true,
|
||||
IncludeAttributes = false,
|
||||
PageSize = 10,
|
||||
CategoryIds = { 10 },
|
||||
TemplateChainContains = { "Pump" },
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
GalaxyObject obj = Assert.Single(reply.Objects);
|
||||
Assert.Equal("Pump_001", obj.TagName);
|
||||
Assert.Empty(obj.Attributes);
|
||||
Assert.Equal(1, reply.TotalObjectCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchy_WithFilteredPaging_ReturnsPostFilterTotal()
|
||||
{
|
||||
GalaxyRepositoryGrpcService service = CreateService(CreateEntry(CreateFilterObjects()));
|
||||
|
||||
DiscoverHierarchyReply first = await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
RootGobjectId = 1,
|
||||
PageSize = 1,
|
||||
CategoryIds = { 10 },
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
DiscoverHierarchyReply second = await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
RootGobjectId = 1,
|
||||
PageSize = 1,
|
||||
PageToken = first.NextPageToken,
|
||||
CategoryIds = { 10 },
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
GalaxyObject firstObject = Assert.Single(first.Objects);
|
||||
GalaxyObject secondObject = Assert.Single(second.Objects);
|
||||
Assert.Equal(2, first.TotalObjectCount);
|
||||
Assert.Equal(2, second.TotalObjectCount);
|
||||
Assert.NotEqual(firstObject.TagName, secondObject.TagName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchy_WithMismatchedFilterToken_ReturnsInvalidArgument()
|
||||
{
|
||||
GalaxyRepositoryGrpcService service = CreateService(CreateEntry(CreateFilterObjects()));
|
||||
DiscoverHierarchyReply first = await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
PageSize = 1,
|
||||
CategoryIds = { 10 },
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
PageSize = 1,
|
||||
PageToken = first.NextPageToken,
|
||||
CategoryIds = { 11 },
|
||||
},
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.InvalidArgument, exception.StatusCode);
|
||||
Assert.Contains("filters", exception.Status.Detail, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchy_WithMissingRoot_ReturnsNotFound()
|
||||
{
|
||||
GalaxyRepositoryGrpcService service = CreateService(CreateEntry(CreateFilterObjects()));
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await service.DiscoverHierarchy(
|
||||
new DiscoverHierarchyRequest
|
||||
{
|
||||
RootTagName = "Missing",
|
||||
},
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.NotFound, exception.StatusCode);
|
||||
}
|
||||
|
||||
private static GalaxyRepositoryGrpcService CreateService(GalaxyHierarchyCacheEntry entry)
|
||||
{
|
||||
GalaxyRepositoryOptions options = new()
|
||||
{
|
||||
ConnectionString = "Server=localhost;Database=ZB;Integrated Security=True;Encrypt=False;",
|
||||
};
|
||||
return new GalaxyRepositoryGrpcService(
|
||||
new global::ZB.MOM.WW.MxGateway.Server.Galaxy.GalaxyRepository(options),
|
||||
new StubGalaxyHierarchyCache(entry),
|
||||
new GalaxyDeployNotifier(),
|
||||
new GatewayRequestIdentityAccessor(),
|
||||
NullLogger<GalaxyRepositoryGrpcService>.Instance);
|
||||
}
|
||||
|
||||
private static GalaxyHierarchyCacheEntry CreateEntry(IReadOnlyList<GalaxyObject> objects)
|
||||
{
|
||||
return GalaxyHierarchyCacheEntry.Empty with
|
||||
{
|
||||
Status = GalaxyCacheStatus.Healthy,
|
||||
Sequence = 7,
|
||||
LastSuccessAt = DateTimeOffset.UtcNow,
|
||||
Objects = objects,
|
||||
Index = GalaxyHierarchyIndex.Build(objects),
|
||||
DashboardSummary = DashboardGalaxySummary.Unknown with
|
||||
{
|
||||
Status = DashboardGalaxyStatus.Healthy,
|
||||
ObjectCount = objects.Count,
|
||||
},
|
||||
ObjectCount = objects.Count,
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<GalaxyObject> CreateObjects(int count)
|
||||
{
|
||||
return Enumerable.Range(1, count)
|
||||
.Select(index => new GalaxyObject
|
||||
{
|
||||
GobjectId = index,
|
||||
TagName = $"Object_{index:000}",
|
||||
BrowseName = $"Object_{index:000}",
|
||||
})
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<GalaxyObject> CreateFilterObjects()
|
||||
{
|
||||
return
|
||||
[
|
||||
new GalaxyObject
|
||||
{
|
||||
GobjectId = 1,
|
||||
TagName = "Area1",
|
||||
ContainedName = "Area1",
|
||||
BrowseName = "Area1",
|
||||
IsArea = true,
|
||||
CategoryId = 13,
|
||||
},
|
||||
new GalaxyObject
|
||||
{
|
||||
GobjectId = 2,
|
||||
TagName = "Line3",
|
||||
ContainedName = "Line3",
|
||||
BrowseName = "Line3",
|
||||
ParentGobjectId = 1,
|
||||
CategoryId = 10,
|
||||
TemplateChain = { "$Line", "$Base" },
|
||||
},
|
||||
new GalaxyObject
|
||||
{
|
||||
GobjectId = 3,
|
||||
TagName = "Pump_001",
|
||||
ContainedName = "Pump",
|
||||
BrowseName = "Pump_001",
|
||||
ParentGobjectId = 2,
|
||||
CategoryId = 10,
|
||||
TemplateChain = { "$Pump", "$Base" },
|
||||
Attributes =
|
||||
{
|
||||
new GalaxyAttribute
|
||||
{
|
||||
AttributeName = "PV",
|
||||
FullTagReference = "Pump_001.PV",
|
||||
IsAlarm = true,
|
||||
IsHistorized = true,
|
||||
SecurityClassification = 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
new GalaxyObject
|
||||
{
|
||||
GobjectId = 4,
|
||||
TagName = "Valve_001",
|
||||
ContainedName = "Valve",
|
||||
BrowseName = "Valve_001",
|
||||
ParentGobjectId = 2,
|
||||
CategoryId = 11,
|
||||
TemplateChain = { "$Valve" },
|
||||
Attributes =
|
||||
{
|
||||
new GalaxyAttribute
|
||||
{
|
||||
AttributeName = "PV",
|
||||
FullTagReference = "Valve_001.PV",
|
||||
},
|
||||
},
|
||||
},
|
||||
new GalaxyObject
|
||||
{
|
||||
GobjectId = 5,
|
||||
TagName = "Other_001",
|
||||
ContainedName = "Other",
|
||||
BrowseName = "Other_001",
|
||||
CategoryId = 10,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
private sealed class StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry current) : IGalaxyHierarchyCache
|
||||
{
|
||||
public GalaxyHierarchyCacheEntry Current { get; } = current;
|
||||
|
||||
public Task RefreshAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task WaitForFirstLoadAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,974 @@
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for Server-021. <c>MxAccessGatewayService.ApplyConstraintsAsync</c> and
|
||||
/// the <c>BulkConstraintPlan</c> / <c>ReadBulkConstraintPlan</c> /
|
||||
/// <c>WriteBulkConstraintPlan</c> / <c>SubscribeBulkConstraintPlan</c> reply-merge
|
||||
/// logic was previously exercised only with an allow-all enforcer, so denial
|
||||
/// filtering, the no-allowed-items short-circuit, and the index-ordered
|
||||
/// denied/allowed interleave were dead code at test time. The fixtures below
|
||||
/// inject a <see cref="PredicateConstraintEnforcer"/> that denies a subset of
|
||||
/// tags or handles, and assert the post-merge reply contents and that the
|
||||
/// session manager is (or is not) invoked.
|
||||
/// </summary>
|
||||
public sealed class MxAccessGatewayServiceConstraintTests
|
||||
{
|
||||
private const string SessionId = "session-constraint";
|
||||
|
||||
// === SubscribeBulk family: AddItemBulk / SubscribeBulk / AdviseItemBulk ===
|
||||
|
||||
/// <summary>
|
||||
/// <c>AddItemBulk</c> with a mix of allowed and denied tags must invoke the
|
||||
/// worker once with only the allowed tags, then splice the denied entries
|
||||
/// back into the reply at their original indices.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_AddItemBulk_WithMixedDenials_InterleavesDeniedAndAllowedInOriginalIndexOrder()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new()
|
||||
{
|
||||
DenyTag = tag => tag == "Tank01.Locked" || tag == "Tank03.Secret",
|
||||
};
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
sessionManager.InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Kind = MxCommandKind.AddItemBulk,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
AddItemBulk = new BulkSubscribeReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
// Worker only sees the two allowed tags — Tank02.Open at original
|
||||
// index 1 and Tank04.Public at original index 3.
|
||||
new SubscribeResult { ServerHandle = 7, TagAddress = "Tank02.Open", ItemHandle = 102, WasSuccessful = true },
|
||||
new SubscribeResult { ServerHandle = 7, TagAddress = "Tank04.Public", ItemHandle = 104, WasSuccessful = true },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateAddItemBulkRequest(7, ["Tank01.Locked", "Tank02.Open", "Tank03.Secret", "Tank04.Public"]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(1, sessionManager.InvokeCount);
|
||||
// Worker saw only the allowed subset, in original order, with denied entries dropped.
|
||||
AddItemBulkCommand forwardedCommand = sessionManager.LastWorkerCommand!.Command.AddItemBulk;
|
||||
Assert.Equal(["Tank02.Open", "Tank04.Public"], forwardedCommand.TagAddresses);
|
||||
// Final reply preserves the original 4-entry index order, with denied entries
|
||||
// at index 0 and 2 and worker-allowed entries at index 1 and 3.
|
||||
BulkSubscribeReply merged = reply.AddItemBulk;
|
||||
Assert.Equal(4, merged.Results.Count);
|
||||
Assert.False(merged.Results[0].WasSuccessful);
|
||||
Assert.Equal("Tank01.Locked", merged.Results[0].TagAddress);
|
||||
Assert.Contains("Tank01.Locked", merged.Results[0].ErrorMessage, StringComparison.Ordinal);
|
||||
Assert.True(merged.Results[1].WasSuccessful);
|
||||
Assert.Equal("Tank02.Open", merged.Results[1].TagAddress);
|
||||
Assert.Equal(102, merged.Results[1].ItemHandle);
|
||||
Assert.False(merged.Results[2].WasSuccessful);
|
||||
Assert.Equal("Tank03.Secret", merged.Results[2].TagAddress);
|
||||
Assert.True(merged.Results[3].WasSuccessful);
|
||||
Assert.Equal("Tank04.Public", merged.Results[3].TagAddress);
|
||||
Assert.Equal(104, merged.Results[3].ItemHandle);
|
||||
// Both denied tags recorded.
|
||||
Assert.Equal(2, enforcer.RecordedDenials.Count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>SubscribeBulk</c> when every tag is denied must short-circuit
|
||||
/// <see cref="BulkConstraintPlan.HasAllowedItems"/> false, return the
|
||||
/// denied-only reply, and never call the session manager.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_SubscribeBulk_WhenAllTagsDenied_DoesNotCallWorkerAndReturnsDeniedReply()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new() { DenyTag = _ => true };
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateSubscribeBulkRequest(7, ["A", "B", "C"]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(0, sessionManager.InvokeCount);
|
||||
Assert.Equal(3, reply.SubscribeBulk.Results.Count);
|
||||
Assert.All(reply.SubscribeBulk.Results, r => Assert.False(r.WasSuccessful));
|
||||
Assert.Equal(["A", "B", "C"], reply.SubscribeBulk.Results.Select(r => r.TagAddress));
|
||||
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>AdviseItemBulk</c> takes handle inputs (not tags) and routes through
|
||||
/// <c>FilterHandleBulkAsync</c> against <c>CheckReadHandleAsync</c>. Partial
|
||||
/// denial must still produce a merged-by-index <c>BulkSubscribeReply</c>.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_AdviseItemBulk_WithMixedHandleDenials_MergesDeniedIntoReply()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new()
|
||||
{
|
||||
DenyReadHandle = (_, itemHandle) => itemHandle == 502,
|
||||
};
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
sessionManager.InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Kind = MxCommandKind.AdviseItemBulk,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
AdviseItemBulk = new BulkSubscribeReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new SubscribeResult { ServerHandle = 7, ItemHandle = 501, WasSuccessful = true },
|
||||
new SubscribeResult { ServerHandle = 7, ItemHandle = 503, WasSuccessful = true },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateAdviseItemBulkRequest(7, [501, 502, 503]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(1, sessionManager.InvokeCount);
|
||||
Assert.Equal([501, 503], sessionManager.LastWorkerCommand!.Command.AdviseItemBulk.ItemHandles);
|
||||
BulkSubscribeReply merged = reply.AdviseItemBulk;
|
||||
Assert.Equal(3, merged.Results.Count);
|
||||
Assert.True(merged.Results[0].WasSuccessful);
|
||||
Assert.Equal(501, merged.Results[0].ItemHandle);
|
||||
Assert.False(merged.Results[1].WasSuccessful);
|
||||
Assert.Equal(502, merged.Results[1].ItemHandle);
|
||||
Assert.True(merged.Results[2].WasSuccessful);
|
||||
Assert.Equal(503, merged.Results[2].ItemHandle);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>SubscribeBulk</c> with an allow-all enforcer must leave the worker reply
|
||||
/// unchanged — the constraint plan is null and no merge occurs. Regression
|
||||
/// guard against accidentally engaging the merge path for the common case.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_SubscribeBulk_WithAllowAllEnforcer_PassesThroughUnchanged()
|
||||
{
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
sessionManager.InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Kind = MxCommandKind.SubscribeBulk,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
SubscribeBulk = new BulkSubscribeReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new SubscribeResult { ServerHandle = 7, TagAddress = "A", ItemHandle = 1, WasSuccessful = true },
|
||||
new SubscribeResult { ServerHandle = 7, TagAddress = "B", ItemHandle = 2, WasSuccessful = true },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
MxAccessGatewayService service = CreateService(sessionManager);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateSubscribeBulkRequest(7, ["A", "B"]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(1, sessionManager.InvokeCount);
|
||||
Assert.Equal(["A", "B"], sessionManager.LastWorkerCommand!.Command.SubscribeBulk.TagAddresses);
|
||||
// Reply identical to worker reply — no synthetic denial rows added.
|
||||
Assert.Equal(2, reply.SubscribeBulk.Results.Count);
|
||||
Assert.All(reply.SubscribeBulk.Results, r => Assert.True(r.WasSuccessful));
|
||||
}
|
||||
|
||||
// === ReadBulk family ===
|
||||
|
||||
/// <summary>
|
||||
/// <c>ReadBulk</c> with a mix of allowed and denied tags merges denied entries
|
||||
/// into the <c>BulkReadReply</c> in original-index order, distinguishable from
|
||||
/// the SubscribeBulk family because the reply slot is
|
||||
/// <c>BulkReadReply</c>, not <c>BulkSubscribeReply</c>.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_ReadBulk_WithMixedDenials_MergesDeniedBulkReadResults()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new()
|
||||
{
|
||||
DenyTag = tag => tag == "Secret.Tag",
|
||||
};
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
sessionManager.InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Kind = MxCommandKind.ReadBulk,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
ReadBulk = new BulkReadReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkReadResult { ServerHandle = 7, TagAddress = "Public.A", WasSuccessful = true },
|
||||
new BulkReadResult { ServerHandle = 7, TagAddress = "Public.B", WasSuccessful = true },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateReadBulkRequest(7, ["Public.A", "Secret.Tag", "Public.B"]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(1, sessionManager.InvokeCount);
|
||||
Assert.Equal(["Public.A", "Public.B"], sessionManager.LastWorkerCommand!.Command.ReadBulk.TagAddresses);
|
||||
BulkReadReply merged = reply.ReadBulk;
|
||||
Assert.Equal(3, merged.Results.Count);
|
||||
Assert.True(merged.Results[0].WasSuccessful);
|
||||
Assert.False(merged.Results[1].WasSuccessful);
|
||||
Assert.Equal("Secret.Tag", merged.Results[1].TagAddress);
|
||||
Assert.True(merged.Results[2].WasSuccessful);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>ReadBulk</c> with all tags denied must short-circuit and produce a
|
||||
/// denied-only <c>BulkReadReply</c> — verifying
|
||||
/// <see cref="MxAccessGatewayService"/>'s <c>ReadBulkConstraintPlan</c>
|
||||
/// <c>CreateDeniedReply</c> path.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_ReadBulk_WhenAllTagsDenied_ShortCircuitsWithDeniedOnlyReply()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new() { DenyTag = _ => true };
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateReadBulkRequest(7, ["X", "Y"]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(0, sessionManager.InvokeCount);
|
||||
Assert.Equal(2, reply.ReadBulk.Results.Count);
|
||||
Assert.All(reply.ReadBulk.Results, r => Assert.False(r.WasSuccessful));
|
||||
Assert.Equal(MxCommandKind.ReadBulk, reply.Kind);
|
||||
}
|
||||
|
||||
// === WriteBulk family: WriteBulk / Write2Bulk / WriteSecuredBulk / WriteSecured2Bulk ===
|
||||
|
||||
/// <summary>
|
||||
/// <c>WriteBulk</c> with one denied handle must drop that entry from the
|
||||
/// forwarded command and splice a denied <c>BulkWriteResult</c> back in at
|
||||
/// the original index.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_WriteBulk_WithDeniedHandle_DropsEntryFromWorkerCallAndMergesDenialIntoReply()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new()
|
||||
{
|
||||
DenyWriteHandle = (_, itemHandle) => itemHandle == 902,
|
||||
};
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
sessionManager.InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Kind = MxCommandKind.WriteBulk,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
WriteBulk = new BulkWriteReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkWriteResult { ServerHandle = 7, ItemHandle = 901, WasSuccessful = true },
|
||||
new BulkWriteResult { ServerHandle = 7, ItemHandle = 903, WasSuccessful = true },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateWriteBulkRequest(7, [901, 902, 903]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(1, sessionManager.InvokeCount);
|
||||
// 902 dropped from forwarded entries; only 901 and 903 reach the worker.
|
||||
WriteBulkCommand forwarded = sessionManager.LastWorkerCommand!.Command.WriteBulk;
|
||||
Assert.Equal([901, 903], forwarded.Entries.Select(e => e.ItemHandle));
|
||||
BulkWriteReply merged = reply.WriteBulk;
|
||||
Assert.Equal(3, merged.Results.Count);
|
||||
Assert.True(merged.Results[0].WasSuccessful);
|
||||
Assert.Equal(901, merged.Results[0].ItemHandle);
|
||||
Assert.False(merged.Results[1].WasSuccessful);
|
||||
Assert.Equal(902, merged.Results[1].ItemHandle);
|
||||
Assert.True(merged.Results[2].WasSuccessful);
|
||||
Assert.Equal(903, merged.Results[2].ItemHandle);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>WriteSecuredBulk</c> exercises a different <c>ReplaceWriteBulkEntries</c>
|
||||
/// switch arm than plain <c>WriteBulk</c>. The merge logic is shared, so a
|
||||
/// full denial here is enough to prove the secured-bulk routing.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_WriteSecuredBulk_WhenAllHandlesDenied_ShortCircuitsWithDeniedOnlyReply()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new() { DenyWriteHandle = (_, _) => true };
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateWriteSecuredBulkRequest(7, [10, 11]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(0, sessionManager.InvokeCount);
|
||||
Assert.Equal(MxCommandKind.WriteSecuredBulk, reply.Kind);
|
||||
Assert.Equal(2, reply.WriteSecuredBulk.Results.Count);
|
||||
Assert.All(reply.WriteSecuredBulk.Results, r => Assert.False(r.WasSuccessful));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests-020: <c>Write2Bulk</c> takes the third <c>GetPayload</c>/<c>SetPayload</c>
|
||||
/// switch arm in <c>WriteBulkConstraintPlan</c>. The merge logic is shared with
|
||||
/// <c>WriteBulk</c>, but a full denial through the <c>CreateDeniedReply</c> path
|
||||
/// proves the <c>Write2Bulk</c> arm of the per-kind <c>SetPayload</c> switch fires
|
||||
/// (and not, say, <c>WriteBulk</c> by mistake) — guarding against a refactor that
|
||||
/// drops or misroutes the <c>Write2Bulk</c> case.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_Write2Bulk_WhenAllHandlesDenied_ShortCircuitsWithDeniedOnlyReply()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new() { DenyWriteHandle = (_, _) => true };
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateWrite2BulkRequest(7, [10, 11]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(0, sessionManager.InvokeCount);
|
||||
Assert.Equal(MxCommandKind.Write2Bulk, reply.Kind);
|
||||
Assert.Equal(2, reply.Write2Bulk.Results.Count);
|
||||
Assert.All(reply.Write2Bulk.Results, r => Assert.False(r.WasSuccessful));
|
||||
// Sibling reply slots must remain empty — pin the SetPayload arm fired
|
||||
// for Write2Bulk and not for one of the other three Write*Bulk kinds.
|
||||
Assert.Empty(reply.WriteBulk?.Results ?? new Google.Protobuf.Collections.RepeatedField<BulkWriteResult>());
|
||||
Assert.Empty(reply.WriteSecuredBulk?.Results ?? new Google.Protobuf.Collections.RepeatedField<BulkWriteResult>());
|
||||
Assert.Empty(reply.WriteSecured2Bulk?.Results ?? new Google.Protobuf.Collections.RepeatedField<BulkWriteResult>());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests-020: <c>WriteSecured2Bulk</c> takes the fourth <c>GetPayload</c>/<c>SetPayload</c>
|
||||
/// switch arm in <c>WriteBulkConstraintPlan</c>. Same reasoning as
|
||||
/// <c>Write2Bulk</c> — assert the <c>WriteSecured2Bulk</c> reply slot is populated
|
||||
/// to prove that arm of the switch fires.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_WriteSecured2Bulk_WhenAllHandlesDenied_ShortCircuitsWithDeniedOnlyReply()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new() { DenyWriteHandle = (_, _) => true };
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateWriteSecured2BulkRequest(7, [10, 11]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(0, sessionManager.InvokeCount);
|
||||
Assert.Equal(MxCommandKind.WriteSecured2Bulk, reply.Kind);
|
||||
Assert.Equal(2, reply.WriteSecured2Bulk.Results.Count);
|
||||
Assert.All(reply.WriteSecured2Bulk.Results, r => Assert.False(r.WasSuccessful));
|
||||
// Sibling reply slots must remain empty — pin the SetPayload arm fired
|
||||
// for WriteSecured2Bulk and not for one of the other three Write*Bulk kinds.
|
||||
Assert.Empty(reply.WriteBulk?.Results ?? new Google.Protobuf.Collections.RepeatedField<BulkWriteResult>());
|
||||
Assert.Empty(reply.Write2Bulk?.Results ?? new Google.Protobuf.Collections.RepeatedField<BulkWriteResult>());
|
||||
Assert.Empty(reply.WriteSecuredBulk?.Results ?? new Google.Protobuf.Collections.RepeatedField<BulkWriteResult>());
|
||||
}
|
||||
|
||||
// === Worker reply-count divergence (Tests-024) ===
|
||||
|
||||
/// <summary>
|
||||
/// Tests-024: <c>WriteBulkConstraintPlan.MergeDeniedInto</c> dequeues from
|
||||
/// <c>allowedResults</c> per non-denied slot via <c>Queue.TryDequeue</c>,
|
||||
/// which silently returns <c>false</c> when the queue is empty. Pin the
|
||||
/// observable behaviour when the worker returns FEWER allowed results than
|
||||
/// the gateway forwarded: the merged reply is truncated — denied entries
|
||||
/// keep their slots, but the trailing allowed slot for which no worker
|
||||
/// result arrived is dropped (no synthetic failure result is fabricated).
|
||||
/// This fixture makes that "silent truncate" behaviour explicit so a future
|
||||
/// change either fills the gap with a synthetic failure or fails this test.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_WriteBulk_WhenWorkerReturnsFewerResultsThanAllowed_MergedReplyIsTruncated()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new()
|
||||
{
|
||||
DenyWriteHandle = (_, itemHandle) => itemHandle == 902,
|
||||
};
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
// Gateway forwards 2 allowed handles (901, 903) but the worker returns only
|
||||
// 1 result. The merge logic should keep denied entry 902 at index 1, place
|
||||
// the single worker result at index 0, and leave index 2 empty (truncate).
|
||||
sessionManager.InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Kind = MxCommandKind.WriteBulk,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
WriteBulk = new BulkWriteReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkWriteResult { ServerHandle = 7, ItemHandle = 901, WasSuccessful = true },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateWriteBulkRequest(7, [901, 902, 903]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(1, sessionManager.InvokeCount);
|
||||
BulkWriteReply merged = reply.WriteBulk;
|
||||
// Current behaviour: the merged reply is shorter than OriginalCount when
|
||||
// the worker under-supplies. Two slots survive — the worker result at
|
||||
// index 0 and the denied entry at index 1 — and the trailing slot is
|
||||
// silently dropped via Queue.TryDequeue returning false.
|
||||
Assert.Equal(2, merged.Results.Count);
|
||||
Assert.True(merged.Results[0].WasSuccessful);
|
||||
Assert.Equal(901, merged.Results[0].ItemHandle);
|
||||
Assert.False(merged.Results[1].WasSuccessful);
|
||||
Assert.Equal(902, merged.Results[1].ItemHandle);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests-024: when the worker returns MORE allowed results than the
|
||||
/// gateway forwarded, the extras must be silently ignored — the merged
|
||||
/// reply length stays at <c>OriginalCount</c>. This pins the
|
||||
/// <c>for index < OriginalCount</c> loop bound so a regression that
|
||||
/// accidentally surfaces extras as trailing results is caught.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_WriteBulk_WhenWorkerReturnsExtraResults_IgnoresExtras()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new()
|
||||
{
|
||||
DenyWriteHandle = (_, itemHandle) => itemHandle == 902,
|
||||
};
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
// Gateway forwards 2 allowed handles (901, 903) but the worker returns 4.
|
||||
sessionManager.InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Kind = MxCommandKind.WriteBulk,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
WriteBulk = new BulkWriteReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkWriteResult { ServerHandle = 7, ItemHandle = 901, WasSuccessful = true },
|
||||
new BulkWriteResult { ServerHandle = 7, ItemHandle = 903, WasSuccessful = true },
|
||||
new BulkWriteResult { ServerHandle = 7, ItemHandle = 999, WasSuccessful = true },
|
||||
new BulkWriteResult { ServerHandle = 7, ItemHandle = 1000, WasSuccessful = true },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreateWriteBulkRequest(7, [901, 902, 903]),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(1, sessionManager.InvokeCount);
|
||||
BulkWriteReply merged = reply.WriteBulk;
|
||||
// Merged reply length stays at OriginalCount (3); the two extra worker
|
||||
// results (item handles 999, 1000) are silently discarded by the
|
||||
// OriginalCount-bounded loop.
|
||||
Assert.Equal(3, merged.Results.Count);
|
||||
Assert.Equal(901, merged.Results[0].ItemHandle);
|
||||
Assert.True(merged.Results[0].WasSuccessful);
|
||||
Assert.Equal(902, merged.Results[1].ItemHandle);
|
||||
Assert.False(merged.Results[1].WasSuccessful);
|
||||
Assert.Equal(903, merged.Results[2].ItemHandle);
|
||||
Assert.True(merged.Results[2].WasSuccessful);
|
||||
Assert.DoesNotContain(merged.Results, r => r.ItemHandle == 999);
|
||||
Assert.DoesNotContain(merged.Results, r => r.ItemHandle == 1000);
|
||||
}
|
||||
|
||||
// === Unary write-handle enforcement (EnforceWriteHandleAsync) ===
|
||||
|
||||
/// <summary>
|
||||
/// Unary <c>Write</c> against a denied (server, item) handle must surface
|
||||
/// <see cref="StatusCode.PermissionDenied"/> via <c>EnforceWriteHandleAsync</c>
|
||||
/// and never reach the session manager.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_Write_WithDeniedHandle_ThrowsPermissionDeniedAndDoesNotCallWorker()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new()
|
||||
{
|
||||
DenyWriteHandle = (serverHandle, itemHandle) => serverHandle == 7 && itemHandle == 42,
|
||||
};
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await service.Invoke(
|
||||
CreateWriteRequest(serverHandle: 7, itemHandle: 42),
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.PermissionDenied, exception.StatusCode);
|
||||
Assert.Equal(0, sessionManager.InvokeCount);
|
||||
Assert.Single(enforcer.RecordedDenials);
|
||||
Assert.Equal("42", enforcer.RecordedDenials[0].Target);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unary <c>WriteSecured</c> against a denied handle takes the same enforce path
|
||||
/// and rejects identically — proving the four-arm switch in
|
||||
/// <c>ApplyConstraintsAsync</c> (Write/Write2/WriteSecured/WriteSecured2) is
|
||||
/// reachable for at least one of the secured kinds.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_WriteSecured_WithDeniedHandle_ThrowsPermissionDenied()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new() { DenyWriteHandle = (_, _) => true };
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await service.Invoke(
|
||||
CreateWriteSecuredRequest(serverHandle: 7, itemHandle: 42),
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.PermissionDenied, exception.StatusCode);
|
||||
Assert.Equal(0, sessionManager.InvokeCount);
|
||||
}
|
||||
|
||||
// === Unary read-tag enforcement (EnforceReadTagAsync via AddItem) ===
|
||||
|
||||
/// <summary>
|
||||
/// Unary <c>AddItem</c> against a denied tag must surface
|
||||
/// <see cref="StatusCode.PermissionDenied"/> via <c>EnforceReadTagAsync</c>
|
||||
/// and never reach the session manager.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_AddItem_WithDeniedTag_ThrowsPermissionDeniedAndDoesNotCallWorker()
|
||||
{
|
||||
PredicateConstraintEnforcer enforcer = new()
|
||||
{
|
||||
DenyTag = tag => tag == "Secret.Tag",
|
||||
};
|
||||
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await service.Invoke(
|
||||
CreateAddItemRequest(serverHandle: 7, tagAddress: "Secret.Tag"),
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.PermissionDenied, exception.StatusCode);
|
||||
Assert.Equal(0, sessionManager.InvokeCount);
|
||||
Assert.Single(enforcer.RecordedDenials);
|
||||
Assert.Equal("Secret.Tag", enforcer.RecordedDenials[0].Target);
|
||||
}
|
||||
|
||||
// === Helpers ===
|
||||
|
||||
private static MxAccessGatewayService CreateService(
|
||||
FakeSessionManager sessionManager,
|
||||
IConstraintEnforcer? constraintEnforcer = null)
|
||||
{
|
||||
return new MxAccessGatewayService(
|
||||
sessionManager,
|
||||
new GatewayRequestIdentityAccessor(),
|
||||
constraintEnforcer ?? new AllowAllConstraintEnforcer(),
|
||||
new MxAccessGrpcRequestValidator(),
|
||||
new MxAccessGrpcMapper(),
|
||||
new FakeEventStreamService(sessionManager),
|
||||
new GatewayMetrics(),
|
||||
NullLogger<MxAccessGatewayService>.Instance,
|
||||
new FakeGatewayAlarmService());
|
||||
}
|
||||
|
||||
private static FakeSessionManager CreateSessionManagerWithSeed()
|
||||
{
|
||||
FakeSessionManager sessionManager = new() { ResolveOnlySeededSessions = true };
|
||||
sessionManager.SeedSession(CreateSession(SessionId));
|
||||
return sessionManager;
|
||||
}
|
||||
|
||||
private static GatewaySession CreateSession(string sessionId)
|
||||
{
|
||||
GatewaySession session = new(
|
||||
sessionId,
|
||||
GatewayContractInfo.DefaultBackendName,
|
||||
"pipe",
|
||||
"nonce",
|
||||
"Operator Key",
|
||||
"operator-session",
|
||||
"client-correlation",
|
||||
TimeSpan.FromSeconds(7),
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromSeconds(10),
|
||||
DateTimeOffset.UtcNow);
|
||||
session.AttachWorkerClient(new FakeWorkerClient());
|
||||
session.MarkReady();
|
||||
return session;
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateAddItemBulkRequest(int serverHandle, IReadOnlyList<string> tags)
|
||||
{
|
||||
AddItemBulkCommand cmd = new() { ServerHandle = serverHandle };
|
||||
cmd.TagAddresses.Add(tags);
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand { Kind = MxCommandKind.AddItemBulk, AddItemBulk = cmd },
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateSubscribeBulkRequest(int serverHandle, IReadOnlyList<string> tags)
|
||||
{
|
||||
SubscribeBulkCommand cmd = new() { ServerHandle = serverHandle };
|
||||
cmd.TagAddresses.Add(tags);
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand { Kind = MxCommandKind.SubscribeBulk, SubscribeBulk = cmd },
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateAdviseItemBulkRequest(int serverHandle, IReadOnlyList<int> itemHandles)
|
||||
{
|
||||
AdviseItemBulkCommand cmd = new() { ServerHandle = serverHandle };
|
||||
cmd.ItemHandles.Add(itemHandles);
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand { Kind = MxCommandKind.AdviseItemBulk, AdviseItemBulk = cmd },
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateReadBulkRequest(int serverHandle, IReadOnlyList<string> tags)
|
||||
{
|
||||
ReadBulkCommand cmd = new() { ServerHandle = serverHandle, TimeoutMs = 1000 };
|
||||
cmd.TagAddresses.Add(tags);
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand { Kind = MxCommandKind.ReadBulk, ReadBulk = cmd },
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateWriteBulkRequest(int serverHandle, IReadOnlyList<int> itemHandles)
|
||||
{
|
||||
WriteBulkCommand cmd = new() { ServerHandle = serverHandle };
|
||||
foreach (int handle in itemHandles)
|
||||
{
|
||||
cmd.Entries.Add(new WriteBulkEntry { ItemHandle = handle, Value = new MxValue { StringValue = "v" } });
|
||||
}
|
||||
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand { Kind = MxCommandKind.WriteBulk, WriteBulk = cmd },
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateWriteSecuredBulkRequest(int serverHandle, IReadOnlyList<int> itemHandles)
|
||||
{
|
||||
WriteSecuredBulkCommand cmd = new() { ServerHandle = serverHandle };
|
||||
foreach (int handle in itemHandles)
|
||||
{
|
||||
cmd.Entries.Add(new WriteSecuredBulkEntry
|
||||
{
|
||||
ItemHandle = handle,
|
||||
CurrentUserId = 1,
|
||||
VerifierUserId = 2,
|
||||
Value = new MxValue { StringValue = "v" },
|
||||
});
|
||||
}
|
||||
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand { Kind = MxCommandKind.WriteSecuredBulk, WriteSecuredBulk = cmd },
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateWrite2BulkRequest(int serverHandle, IReadOnlyList<int> itemHandles)
|
||||
{
|
||||
Write2BulkCommand cmd = new() { ServerHandle = serverHandle };
|
||||
foreach (int handle in itemHandles)
|
||||
{
|
||||
cmd.Entries.Add(new Write2BulkEntry
|
||||
{
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { StringValue = "v" },
|
||||
TimestampValue = new MxValue { Int64Value = 1234567890L },
|
||||
});
|
||||
}
|
||||
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand { Kind = MxCommandKind.Write2Bulk, Write2Bulk = cmd },
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateWriteSecured2BulkRequest(int serverHandle, IReadOnlyList<int> itemHandles)
|
||||
{
|
||||
WriteSecured2BulkCommand cmd = new() { ServerHandle = serverHandle };
|
||||
foreach (int handle in itemHandles)
|
||||
{
|
||||
cmd.Entries.Add(new WriteSecured2BulkEntry
|
||||
{
|
||||
ItemHandle = handle,
|
||||
CurrentUserId = 1,
|
||||
VerifierUserId = 2,
|
||||
Value = new MxValue { StringValue = "v" },
|
||||
TimestampValue = new MxValue { Int64Value = 1234567890L },
|
||||
});
|
||||
}
|
||||
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand { Kind = MxCommandKind.WriteSecured2Bulk, WriteSecured2Bulk = cmd },
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateWriteRequest(int serverHandle, int itemHandle)
|
||||
{
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.Write,
|
||||
Write = new WriteCommand
|
||||
{
|
||||
ServerHandle = serverHandle,
|
||||
ItemHandle = itemHandle,
|
||||
Value = new MxValue { StringValue = "v" },
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateWriteSecuredRequest(int serverHandle, int itemHandle)
|
||||
{
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.WriteSecured,
|
||||
WriteSecured = new WriteSecuredCommand
|
||||
{
|
||||
ServerHandle = serverHandle,
|
||||
ItemHandle = itemHandle,
|
||||
CurrentUserId = 1,
|
||||
VerifierUserId = 2,
|
||||
Value = new MxValue { StringValue = "v" },
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreateAddItemRequest(int serverHandle, string tagAddress)
|
||||
{
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AddItem,
|
||||
AddItem = new AddItemCommand
|
||||
{
|
||||
ServerHandle = serverHandle,
|
||||
ItemDefinition = tagAddress,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// FakeSessionManager / FakeEventStreamService / FakeWorkerClient mirror the
|
||||
// implementations in MxAccessGatewayServiceTests; the duplication is intentional
|
||||
// so the constraint tests are self-contained and changes to the existing fakes
|
||||
// don't accidentally couple the two suites.
|
||||
private sealed class FakeSessionManager : ISessionManager
|
||||
{
|
||||
private readonly Dictionary<string, GatewaySession> seededSessions = new(StringComparer.Ordinal);
|
||||
|
||||
public bool ResolveOnlySeededSessions { get; init; }
|
||||
|
||||
public WorkerCommand? LastWorkerCommand { get; private set; }
|
||||
|
||||
public int InvokeCount { get; private set; }
|
||||
|
||||
public WorkerCommandReply InvokeReply { get; set; } = new()
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Kind = MxCommandKind.Ping,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
},
|
||||
};
|
||||
|
||||
public List<WorkerEvent> Events { get; } = [];
|
||||
|
||||
public void SeedSession(GatewaySession session) => seededSessions[session.SessionId] = session;
|
||||
|
||||
public Task<GatewaySession> OpenSessionAsync(
|
||||
SessionOpenRequest request,
|
||||
string? clientIdentity,
|
||||
CancellationToken cancellationToken) =>
|
||||
Task.FromResult(seededSessions.Values.First());
|
||||
|
||||
public bool TryGetSession(string sessionId, out GatewaySession session)
|
||||
{
|
||||
if (seededSessions.TryGetValue(sessionId, out GatewaySession? seeded))
|
||||
{
|
||||
session = seeded;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (ResolveOnlySeededSessions)
|
||||
{
|
||||
session = null!;
|
||||
return false;
|
||||
}
|
||||
|
||||
session = CreateFallbackSession(sessionId);
|
||||
return true;
|
||||
}
|
||||
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
string sessionId,
|
||||
WorkerCommand command,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
InvokeCount++;
|
||||
LastWorkerCommand = command;
|
||||
return Task.FromResult(InvokeReply);
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
string sessionId,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (WorkerEvent ev in Events)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await Task.Yield();
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
|
||||
public Task<SessionCloseResult> CloseSessionAsync(
|
||||
string sessionId,
|
||||
CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false));
|
||||
|
||||
public Task<int> CloseExpiredLeasesAsync(
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken) => Task.FromResult(0);
|
||||
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
private static GatewaySession CreateFallbackSession(string sessionId)
|
||||
{
|
||||
GatewaySession session = new(
|
||||
sessionId,
|
||||
GatewayContractInfo.DefaultBackendName,
|
||||
"pipe",
|
||||
"nonce",
|
||||
"Operator Key",
|
||||
"operator-session",
|
||||
"client-correlation",
|
||||
TimeSpan.FromSeconds(7),
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromSeconds(10),
|
||||
DateTimeOffset.UtcNow);
|
||||
session.AttachWorkerClient(new FakeWorkerClient());
|
||||
session.MarkReady();
|
||||
return session;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeEventStreamService(FakeSessionManager sessionManager) : IEventStreamService
|
||||
{
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync(
|
||||
StreamEventsRequest request,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (WorkerEvent ev in sessionManager.Events)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await Task.Yield();
|
||||
yield return ev.Event;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeWorkerClient : IWorkerClient
|
||||
{
|
||||
public string SessionId { get; } = MxAccessGatewayServiceConstraintTests.SessionId;
|
||||
|
||||
public int? ProcessId { get; } = 1234;
|
||||
|
||||
public WorkerClientState State { get; } = WorkerClientState.Ready;
|
||||
|
||||
public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.UtcNow;
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
WorkerCommand command,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken) => Task.FromResult(new WorkerCommandReply());
|
||||
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
}
|
||||
|
||||
public Task ShutdownAsync(TimeSpan timeout, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public void Kill(string reason)
|
||||
{
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,636 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
|
||||
|
||||
public sealed class MxAccessGatewayServiceTests
|
||||
{
|
||||
/// <summary>Verifies that OpenSession returns correct session details for a valid request.</summary>
|
||||
[Fact]
|
||||
public async Task OpenSession_WithValidRequest_ReturnsSessionDetails()
|
||||
{
|
||||
GatewayRequestIdentityAccessor identityAccessor = new();
|
||||
FakeSessionManager sessionManager = new()
|
||||
{
|
||||
OpenSessionResult = CreateSession("session-1", processId: 4321),
|
||||
};
|
||||
MxAccessGatewayService service = CreateService(sessionManager, identityAccessor);
|
||||
|
||||
using IDisposable identityScope = identityAccessor.Push(CreateIdentity());
|
||||
OpenSessionReply reply = await service.OpenSession(
|
||||
new OpenSessionRequest
|
||||
{
|
||||
ClientSessionName = "operator-session",
|
||||
CommandTimeout = Duration.FromTimeSpan(TimeSpan.FromSeconds(7)),
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal("session-1", reply.SessionId);
|
||||
Assert.Equal(GatewayContractInfo.DefaultBackendName, reply.BackendName);
|
||||
Assert.Equal(4321, reply.WorkerProcessId);
|
||||
Assert.Equal(GatewayContractInfo.WorkerProtocolVersion, reply.WorkerProtocolVersion);
|
||||
Assert.Equal(GatewayContractInfo.GatewayProtocolVersion, reply.GatewayProtocolVersion);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
|
||||
Assert.Contains("unary-invoke", reply.Capabilities);
|
||||
Assert.Equal("Operator Key", sessionManager.LastClientIdentity);
|
||||
Assert.Equal("operator-session", sessionManager.LastOpenRequest?.ClientSessionName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that Invoke maps a genuinely missing session to NotFound via the
|
||||
/// service's own <c>ResolveSession</c> lookup. No <c>InvokeException</c> is
|
||||
/// injected — <see cref="FakeSessionManager.ResolveOnlySeededSessions"/> makes
|
||||
/// <c>TryGetSession</c> return false, so this test fails if the service drops
|
||||
/// its missing-session check rather than passing for the wrong reason.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_WhenSessionMissing_ThrowsNotFound()
|
||||
{
|
||||
FakeSessionManager sessionManager = new() { ResolveOnlySeededSessions = true };
|
||||
MxAccessGatewayService service = CreateService(sessionManager);
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await service.Invoke(
|
||||
CreatePingRequest("session-missing"),
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.NotFound, exception.StatusCode);
|
||||
Assert.Contains("session-missing", exception.Status.Detail, StringComparison.Ordinal);
|
||||
// The service must reject before delegating to the session manager.
|
||||
Assert.Equal(0, sessionManager.InvokeCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that Invoke resolves a session that was seeded into the session
|
||||
/// manager when <see cref="FakeSessionManager.ResolveOnlySeededSessions"/> is on,
|
||||
/// confirming the missing-session test above is gated on a real lookup.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Invoke_WhenSessionSeeded_ResolvesAndInvokes()
|
||||
{
|
||||
FakeSessionManager sessionManager = new() { ResolveOnlySeededSessions = true };
|
||||
sessionManager.SeedSession(CreateSession("session-1", processId: 1234));
|
||||
MxAccessGatewayService service = CreateService(sessionManager);
|
||||
|
||||
MxCommandReply reply = await service.Invoke(
|
||||
CreatePingRequest("session-1"),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
|
||||
Assert.Equal(1, sessionManager.InvokeCount);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that Invoke throws InvalidArgument and does not invoke the session manager when payload is mismatched.</summary>
|
||||
[Fact]
|
||||
public async Task Invoke_WithMismatchedPayload_ThrowsInvalidArgumentAndDoesNotCallSessionManager()
|
||||
{
|
||||
FakeSessionManager sessionManager = new();
|
||||
MxAccessGatewayService service = CreateService(sessionManager);
|
||||
MxCommandRequest request = new()
|
||||
{
|
||||
SessionId = "session-1",
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AddItem,
|
||||
Ping = new PingCommand { Message = "wrong-payload" },
|
||||
},
|
||||
};
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await service.Invoke(request, new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.InvalidArgument, exception.StatusCode);
|
||||
Assert.Equal(0, sessionManager.InvokeCount);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that Invoke returns HResult status and method payload from worker reply.</summary>
|
||||
[Fact]
|
||||
public async Task Invoke_WithWorkerReply_ReturnsHresultStatusAndMethodPayload()
|
||||
{
|
||||
const int hresult = unchecked((int)0x80004005);
|
||||
FakeSessionManager sessionManager = new()
|
||||
{
|
||||
InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = "session-1",
|
||||
CorrelationId = "worker-correlation",
|
||||
Kind = MxCommandKind.AddItem,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
Hresult = hresult,
|
||||
AddItem = new AddItemReply { ItemHandle = 42 },
|
||||
DiagnosticMessage = "mxaccess diagnostic",
|
||||
},
|
||||
},
|
||||
};
|
||||
sessionManager.InvokeReply.Reply.Statuses.Add(new MxStatusProxy
|
||||
{
|
||||
Success = 0,
|
||||
Category = MxStatusCategory.SoftwareError,
|
||||
Detail = 1001,
|
||||
DiagnosticText = "status detail",
|
||||
});
|
||||
MxAccessGatewayService service = CreateService(sessionManager);
|
||||
MxCommandRequest request = new()
|
||||
{
|
||||
SessionId = "session-1",
|
||||
ClientCorrelationId = "client-correlation",
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AddItem,
|
||||
AddItem = new AddItemCommand
|
||||
{
|
||||
ServerHandle = 12,
|
||||
ItemDefinition = "Galaxy.Tag.Value",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
MxCommandReply reply = await service.Invoke(request, new TestServerCallContext());
|
||||
|
||||
Assert.Equal(MxCommandKind.AddItem, sessionManager.LastWorkerCommand?.Command.Kind);
|
||||
Assert.Equal("Galaxy.Tag.Value", sessionManager.LastWorkerCommand?.Command.AddItem.ItemDefinition);
|
||||
Assert.NotNull(sessionManager.LastWorkerCommand?.EnqueueTimestamp);
|
||||
Assert.Equal(hresult, reply.Hresult);
|
||||
Assert.Equal(42, reply.AddItem.ItemHandle);
|
||||
Assert.Equal("status detail", Assert.Single(reply.Statuses).DiagnosticText);
|
||||
Assert.Equal("mxaccess diagnostic", reply.DiagnosticMessage);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that StreamEvents writes only events after the specified worker sequence.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEvents_WithAfterSequence_WritesOnlyLaterEvents()
|
||||
{
|
||||
FakeSessionManager sessionManager = new();
|
||||
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
|
||||
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 2));
|
||||
MxAccessGatewayService service = CreateService(sessionManager);
|
||||
RecordingServerStreamWriter<MxEvent> writer = new();
|
||||
|
||||
await service.StreamEvents(
|
||||
new StreamEventsRequest
|
||||
{
|
||||
SessionId = "session-1",
|
||||
AfterWorkerSequence = 1,
|
||||
},
|
||||
writer,
|
||||
new TestServerCallContext());
|
||||
|
||||
MxEvent writtenEvent = Assert.Single(writer.Messages);
|
||||
Assert.Equal((ulong)2, writtenEvent.WorkerSequence);
|
||||
Assert.Equal("session-1", sessionManager.LastReadEventsSessionId);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that StreamEvents records send duration metrics when an event is written.</summary>
|
||||
[Fact]
|
||||
public async Task StreamEvents_WhenEventIsWritten_RecordsSendDuration()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
using MeterListener listener = new();
|
||||
List<string> families = [];
|
||||
listener.InstrumentPublished = (instrument, meterListener) =>
|
||||
{
|
||||
if (instrument.Meter.Name == GatewayMetrics.MeterName
|
||||
&& instrument.Name == "mxgateway.events.stream_send.duration")
|
||||
{
|
||||
meterListener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
listener.SetMeasurementEventCallback<double>(
|
||||
(instrument, measurement, tags, _) =>
|
||||
{
|
||||
if (instrument.Name != "mxgateway.events.stream_send.duration")
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (KeyValuePair<string, object?> tag in tags)
|
||||
{
|
||||
if (tag.Key == "family" && tag.Value is string family)
|
||||
{
|
||||
families.Add(family);
|
||||
}
|
||||
}
|
||||
});
|
||||
listener.Start();
|
||||
FakeSessionManager sessionManager = new();
|
||||
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 2));
|
||||
MxAccessGatewayService service = CreateService(sessionManager, metrics: metrics);
|
||||
RecordingServerStreamWriter<MxEvent> writer = new();
|
||||
|
||||
await service.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = "session-1" },
|
||||
writer,
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal([MxEventFamily.OnDataChange.ToString()], families);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that CloseSession throws InvalidArgument when session ID is blank.</summary>
|
||||
[Fact]
|
||||
public async Task CloseSession_WithBlankSessionId_ThrowsInvalidArgument()
|
||||
{
|
||||
MxAccessGatewayService service = CreateService(new FakeSessionManager());
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await service.CloseSession(
|
||||
new CloseSessionRequest(),
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.InvalidArgument, exception.StatusCode);
|
||||
}
|
||||
|
||||
// ===== AcknowledgeAlarm + StreamAlarms handler contract =====
|
||||
//
|
||||
// AcknowledgeAlarm validates alarm_full_reference then delegates to the
|
||||
// session-less IGatewayAlarmService; StreamAlarms forwards the central
|
||||
// alarm feed. CreateService injects FakeGatewayAlarmService.
|
||||
|
||||
/// <summary>Verifies AcknowledgeAlarm rejects an empty alarm_full_reference.</summary>
|
||||
[Fact]
|
||||
public async Task AcknowledgeAlarm_WithMissingAlarmReference_ThrowsInvalidArgument()
|
||||
{
|
||||
MxAccessGatewayService service = CreateService(new FakeSessionManager());
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await service.AcknowledgeAlarm(
|
||||
new AcknowledgeAlarmRequest { OperatorUser = "alice" },
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.InvalidArgument, exception.StatusCode);
|
||||
}
|
||||
|
||||
/// <summary>Verifies AcknowledgeAlarm delegates a valid request to the alarm service.</summary>
|
||||
[Fact]
|
||||
public async Task AcknowledgeAlarm_WithValidRequest_DelegatesToAlarmService()
|
||||
{
|
||||
MxAccessGatewayService service = CreateService(new FakeSessionManager());
|
||||
|
||||
AcknowledgeAlarmReply reply = await service.AcknowledgeAlarm(
|
||||
new AcknowledgeAlarmRequest
|
||||
{
|
||||
ClientCorrelationId = "corr-1",
|
||||
AlarmFullReference = "Galaxy!Area.Tank01.Level.HiHi",
|
||||
Comment = "investigating",
|
||||
OperatorUser = "alice",
|
||||
},
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
|
||||
Assert.Equal("corr-1", reply.CorrelationId);
|
||||
}
|
||||
|
||||
/// <summary>Verifies StreamAlarms forwards the central alarm feed, ending with snapshot_complete.</summary>
|
||||
[Fact]
|
||||
public async Task StreamAlarms_ForwardsTheCentralAlarmFeed()
|
||||
{
|
||||
MxAccessGatewayService service = CreateService(new FakeSessionManager());
|
||||
RecordingServerStreamWriter<AlarmFeedMessage> sink = new();
|
||||
|
||||
await service.StreamAlarms(
|
||||
new StreamAlarmsRequest(),
|
||||
sink,
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Contains(
|
||||
sink.Messages,
|
||||
message => message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.SnapshotComplete);
|
||||
}
|
||||
|
||||
/// <summary>Verifies OpenSession advertises the alarm RPC capability strings.</summary>
|
||||
[Fact]
|
||||
public async Task OpenSession_AdvertisesAlarmRpcCapabilities()
|
||||
{
|
||||
FakeSessionManager sessionManager = new();
|
||||
GatewayRequestIdentityAccessor identityAccessor = new();
|
||||
MxAccessGatewayService service = CreateService(sessionManager, identityAccessor);
|
||||
|
||||
using IDisposable identityScope = identityAccessor.Push(CreateIdentity());
|
||||
|
||||
OpenSessionReply reply = await service.OpenSession(
|
||||
new OpenSessionRequest(),
|
||||
new TestServerCallContext());
|
||||
|
||||
Assert.Contains("unary-acknowledge-alarm", reply.Capabilities);
|
||||
Assert.Contains("server-stream-active-alarms", reply.Capabilities);
|
||||
}
|
||||
|
||||
private static MxAccessGatewayService CreateService(
|
||||
FakeSessionManager sessionManager,
|
||||
IGatewayRequestIdentityAccessor? identityAccessor = null,
|
||||
GatewayMetrics? metrics = null,
|
||||
IConstraintEnforcer? constraintEnforcer = null)
|
||||
{
|
||||
return new MxAccessGatewayService(
|
||||
sessionManager,
|
||||
identityAccessor ?? new GatewayRequestIdentityAccessor(),
|
||||
constraintEnforcer ?? new AllowAllConstraintEnforcer(),
|
||||
new MxAccessGrpcRequestValidator(),
|
||||
new MxAccessGrpcMapper(),
|
||||
new FakeEventStreamService(sessionManager),
|
||||
metrics ?? new GatewayMetrics(),
|
||||
NullLogger<MxAccessGatewayService>.Instance,
|
||||
new FakeGatewayAlarmService());
|
||||
}
|
||||
|
||||
private static ApiKeyIdentity CreateIdentity()
|
||||
{
|
||||
return new ApiKeyIdentity(
|
||||
KeyId: "operator01",
|
||||
KeyPrefix: "mxgw_operator01",
|
||||
DisplayName: "Operator Key",
|
||||
Scopes: new HashSet<string>(StringComparer.Ordinal));
|
||||
}
|
||||
|
||||
private static GatewaySession CreateSession(
|
||||
string sessionId,
|
||||
int processId)
|
||||
{
|
||||
GatewaySession session = new(
|
||||
sessionId,
|
||||
GatewayContractInfo.DefaultBackendName,
|
||||
"pipe",
|
||||
"nonce",
|
||||
"Operator Key",
|
||||
"operator-session",
|
||||
"client-correlation",
|
||||
TimeSpan.FromSeconds(7),
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromSeconds(10),
|
||||
DateTimeOffset.UtcNow);
|
||||
session.AttachWorkerClient(new FakeWorkerClient(processId));
|
||||
session.MarkReady();
|
||||
|
||||
return session;
|
||||
}
|
||||
|
||||
private static MxCommandRequest CreatePingRequest(string sessionId)
|
||||
{
|
||||
return new MxCommandRequest
|
||||
{
|
||||
SessionId = sessionId,
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.Ping,
|
||||
Ping = new PingCommand { Message = "ping" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static WorkerEvent CreateWorkerEvent(
|
||||
string sessionId,
|
||||
ulong workerSequence)
|
||||
{
|
||||
return new WorkerEvent
|
||||
{
|
||||
Event = new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
SessionId = sessionId,
|
||||
WorkerSequence = workerSequence,
|
||||
OnDataChange = new OnDataChangeEvent(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class FakeSessionManager : ISessionManager
|
||||
{
|
||||
private readonly Dictionary<string, GatewaySession> seededSessions = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>The session to return from OpenSessionAsync.</summary>
|
||||
public GatewaySession? OpenSessionResult { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When true, <see cref="TryGetSession"/> only resolves sessions that have been
|
||||
/// explicitly seeded via <see cref="SeedSession"/> (or <see cref="OpenSessionResult"/>),
|
||||
/// and returns false for any other id. This exercises the gateway service's own
|
||||
/// missing-session handling instead of masking it with a synthesized session.
|
||||
/// </summary>
|
||||
public bool ResolveOnlySeededSessions { get; init; }
|
||||
|
||||
/// <summary>Registers a session so <see cref="TryGetSession"/> resolves its id.</summary>
|
||||
/// <param name="session">Session to register by its <see cref="GatewaySession.SessionId"/>.</param>
|
||||
public void SeedSession(GatewaySession session)
|
||||
{
|
||||
seededSessions[session.SessionId] = session;
|
||||
}
|
||||
|
||||
/// <summary>The last OpenSessionAsync request captured.</summary>
|
||||
public SessionOpenRequest? LastOpenRequest { get; private set; }
|
||||
|
||||
/// <summary>The last client identity passed to OpenSessionAsync.</summary>
|
||||
public string? LastClientIdentity { get; private set; }
|
||||
|
||||
/// <summary>The last session ID passed to ReadEventsAsync.</summary>
|
||||
public string? LastReadEventsSessionId { get; private set; }
|
||||
|
||||
/// <summary>The last worker command passed to InvokeAsync.</summary>
|
||||
public WorkerCommand? LastWorkerCommand { get; private set; }
|
||||
|
||||
/// <summary>The reply to return from InvokeAsync.</summary>
|
||||
public WorkerCommandReply InvokeReply { get; init; } = new()
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = "session-1",
|
||||
Kind = MxCommandKind.Ping,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
},
|
||||
};
|
||||
|
||||
/// <summary>The exception to throw from InvokeAsync.</summary>
|
||||
public Exception? InvokeException { get; init; }
|
||||
|
||||
/// <summary>The number of times InvokeAsync was called.</summary>
|
||||
public int InvokeCount { get; private set; }
|
||||
|
||||
/// <summary>The events to return from ReadEventsAsync.</summary>
|
||||
public List<WorkerEvent> Events { get; } = [];
|
||||
|
||||
/// <summary>Records the session ID passed to ReadEventsAsync.</summary>
|
||||
/// <param name="sessionId">Identifier of the session.</param>
|
||||
public void RecordReadEventsSessionId(string sessionId)
|
||||
{
|
||||
LastReadEventsSessionId = sessionId;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<GatewaySession> OpenSessionAsync(
|
||||
SessionOpenRequest request,
|
||||
string? clientIdentity,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
LastOpenRequest = request;
|
||||
LastClientIdentity = clientIdentity;
|
||||
|
||||
return Task.FromResult(OpenSessionResult ?? CreateSession("session-1", processId: 1234));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool TryGetSession(
|
||||
string sessionId,
|
||||
out GatewaySession session)
|
||||
{
|
||||
if (seededSessions.TryGetValue(sessionId, out GatewaySession? seeded))
|
||||
{
|
||||
session = seeded;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (ResolveOnlySeededSessions)
|
||||
{
|
||||
session = null!;
|
||||
return false;
|
||||
}
|
||||
|
||||
session = OpenSessionResult ?? CreateSession(sessionId, processId: 1234);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
string sessionId,
|
||||
WorkerCommand command,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
InvokeCount++;
|
||||
LastWorkerCommand = command;
|
||||
|
||||
if (InvokeException is not null)
|
||||
{
|
||||
throw InvokeException;
|
||||
}
|
||||
|
||||
return Task.FromResult(InvokeReply);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
string sessionId,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
LastReadEventsSessionId = sessionId;
|
||||
foreach (WorkerEvent workerEvent in Events)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await Task.Yield();
|
||||
yield return workerEvent;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<SessionCloseResult> CloseSessionAsync(
|
||||
string sessionId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<int> CloseExpiredLeasesAsync(
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeEventStreamService(FakeSessionManager sessionManager) : IEventStreamService
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync(
|
||||
StreamEventsRequest request,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
sessionManager.RecordReadEventsSessionId(request.SessionId);
|
||||
foreach (WorkerEvent workerEvent in sessionManager.Events)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await Task.Yield();
|
||||
if (workerEvent.Event.WorkerSequence <= request.AfterWorkerSequence)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return workerEvent.Event;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeWorkerClient(int processId) : IWorkerClient
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public string SessionId { get; } = "session-1";
|
||||
|
||||
/// <inheritdoc />
|
||||
public int? ProcessId { get; } = processId;
|
||||
|
||||
/// <inheritdoc />
|
||||
public WorkerClientState State { get; } = WorkerClientState.Ready;
|
||||
|
||||
/// <inheritdoc />
|
||||
public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.UtcNow;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
WorkerCommand command,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new WorkerCommandReply());
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ShutdownAsync(
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Kill(string reason)
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
|
||||
|
||||
public sealed class MxAccessGrpcMapperTests
|
||||
{
|
||||
/// <summary>Verifies that command mapping clones payloads to isolate them across process boundaries.</summary>
|
||||
[Fact]
|
||||
public void MapCommand_ClonesMethodSpecificPayloadForWorkerBoundary()
|
||||
{
|
||||
MxAccessGrpcMapper mapper = new();
|
||||
MxCommandRequest request = new()
|
||||
{
|
||||
SessionId = "session-1",
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.Write,
|
||||
Write = new WriteCommand
|
||||
{
|
||||
ServerHandle = 10,
|
||||
ItemHandle = 20,
|
||||
UserId = 30,
|
||||
Value = new MxValue
|
||||
{
|
||||
DataType = MxDataType.String,
|
||||
StringValue = "value",
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
WorkerCommand workerCommand = mapper.MapCommand(request);
|
||||
request.Command.Write.Value.StringValue = "changed";
|
||||
|
||||
Assert.Equal(MxCommandKind.Write, workerCommand.Command.Kind);
|
||||
Assert.Equal("value", workerCommand.Command.Write.Value.StringValue);
|
||||
Assert.NotNull(workerCommand.EnqueueTimestamp);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that command reply mapping preserves HRESULT and status information.</summary>
|
||||
[Fact]
|
||||
public void MapCommandReply_PreservesHresultStatusesAndPayload()
|
||||
{
|
||||
const int hresult = unchecked((int)0x80070005);
|
||||
WorkerCommandReply workerReply = new()
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = "session-1",
|
||||
Kind = MxCommandKind.Register,
|
||||
ProtocolStatus = MxAccessGrpcMapper.Ok(),
|
||||
Hresult = hresult,
|
||||
Register = new RegisterReply { ServerHandle = 50 },
|
||||
},
|
||||
};
|
||||
workerReply.Reply.Statuses.Add(new MxStatusProxy
|
||||
{
|
||||
Success = 0,
|
||||
Category = MxStatusCategory.SecurityError,
|
||||
DiagnosticText = "denied",
|
||||
});
|
||||
|
||||
MxCommandReply publicReply = new MxAccessGrpcMapper().MapCommandReply(workerReply);
|
||||
|
||||
Assert.Equal(hresult, publicReply.Hresult);
|
||||
Assert.Equal(50, publicReply.Register.ServerHandle);
|
||||
Assert.Equal("denied", Assert.Single(publicReply.Statuses).DiagnosticText);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a missing worker reply returns a protocol violation status.</summary>
|
||||
[Fact]
|
||||
public void MapCommandReply_WhenWorkerReplyMissing_ReturnsProtocolViolationReply()
|
||||
{
|
||||
MxCommandReply publicReply = new MxAccessGrpcMapper().MapCommandReply(new WorkerCommandReply());
|
||||
|
||||
Assert.Equal(ProtocolStatusCode.ProtocolViolation, publicReply.ProtocolStatus.Code);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
|
||||
|
||||
/// <summary>
|
||||
/// Concurrency and disposal regression tests for <see cref="GatewaySession"/>.
|
||||
/// Server-015 and Server-016 audited the split lock discipline between
|
||||
/// <c>_syncRoot</c> (state transitions) and <c>_closeLock</c> (close serialization)
|
||||
/// and the un-gated <c>DisposeAsync</c>; these tests pin the post-fix behavior.
|
||||
/// </summary>
|
||||
public sealed class GatewaySessionTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Server-015 regression. A <c>TransitionTo(Ready)</c> issued after
|
||||
/// <see cref="GatewaySession.CloseAsync"/> has set <see cref="SessionState.Closing"/>
|
||||
/// must not flip the session back to <see cref="SessionState.Ready"/>. The
|
||||
/// blocking worker shutdown keeps <c>CloseAsync</c> parked between the
|
||||
/// <c>Closing</c> write and the <c>Closed</c> write, which is precisely the
|
||||
/// window the audit identified.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task TransitionTo_AfterCloseStarted_DoesNotOverwriteClosing()
|
||||
{
|
||||
BlockingShutdownWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
|
||||
Task<SessionCloseResult> closeTask = session.CloseAsync("test-close", CancellationToken.None);
|
||||
await workerClient.WaitForShutdownStartAsync();
|
||||
|
||||
// Close has set _state = Closing under _syncRoot and is parked inside
|
||||
// worker.ShutdownAsync. A concurrent transition (e.g. a late
|
||||
// SessionWorkerClientFactory lifecycle callback) must not revive the session.
|
||||
Assert.Equal(SessionState.Closing, session.State);
|
||||
session.TransitionTo(SessionState.Ready);
|
||||
Assert.Equal(SessionState.Closing, session.State);
|
||||
|
||||
workerClient.ReleaseShutdown();
|
||||
SessionCloseResult result = await closeTask;
|
||||
|
||||
Assert.Equal(SessionState.Closed, result.FinalState);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
|
||||
await session.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Server-015 regression. Once <see cref="GatewaySession.CloseAsync"/> finishes,
|
||||
/// <see cref="GatewaySession.MarkFaulted"/> must not be able to move the
|
||||
/// session out of <see cref="SessionState.Closed"/> either — the close path's
|
||||
/// terminal write goes through the same <c>_syncRoot</c> the rest of the state
|
||||
/// machine uses, so the existing "Closed is terminal" invariant holds.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task MarkFaulted_AfterCloseCompletes_DoesNotResurrectSession()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
|
||||
await session.CloseAsync("test-close", CancellationToken.None);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
|
||||
session.MarkFaulted("late-fault");
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
|
||||
await session.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Server-028 regression. A <see cref="GatewaySession.MarkFaulted"/> issued
|
||||
/// while <see cref="GatewaySession.CloseAsync"/> is parked between its
|
||||
/// <c>Closing</c> and <c>Closed</c> writes must not break the close path's
|
||||
/// terminal contract: the in-flight close runs to <c>Closed</c>, the fault
|
||||
/// reason is preserved on <see cref="GatewaySession.FinalFault"/>, and the
|
||||
/// session does not get stuck in <see cref="SessionState.Faulted"/>. The
|
||||
/// state machine documents "Closing only allows a transition to Closed or
|
||||
/// Faulted" — this test pins the resolved end state so a future tightening
|
||||
/// of <c>MarkFaulted</c> cannot silently regress it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task MarkFaulted_DuringInFlightClose_PreservesFaultButYieldsToClose()
|
||||
{
|
||||
BlockingShutdownWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
|
||||
Task<SessionCloseResult> closeTask = session.CloseAsync("test-close", CancellationToken.None);
|
||||
await workerClient.WaitForShutdownStartAsync();
|
||||
|
||||
// Close has set _state = Closing under _syncRoot and is parked inside
|
||||
// worker.ShutdownAsync. Fault the session from another thread while parked.
|
||||
Assert.Equal(SessionState.Closing, session.State);
|
||||
session.MarkFaulted("concurrent-fault");
|
||||
|
||||
workerClient.ReleaseShutdown();
|
||||
SessionCloseResult result = await closeTask;
|
||||
|
||||
// Close still wins — Closed is terminal — but the fault reason is preserved
|
||||
// so observers see the original cause once the session settles.
|
||||
Assert.Equal(SessionState.Closed, result.FinalState);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
Assert.Equal("concurrent-fault", session.FinalFault);
|
||||
|
||||
await session.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Server-016 regression. <see cref="GatewaySession.DisposeAsync"/> must wait
|
||||
/// for an in-flight <see cref="GatewaySession.CloseAsync"/> before disposing
|
||||
/// its semaphore. Without the fix, the close's <c>_closeLock.Release()</c>
|
||||
/// would race the dispose and raise <see cref="ObjectDisposedException"/>.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DisposeAsync_WhileCloseInFlight_WaitsForCloseAndDoesNotThrow()
|
||||
{
|
||||
BlockingShutdownWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
|
||||
Task<SessionCloseResult> closeTask = session.CloseAsync("test-close", CancellationToken.None);
|
||||
await workerClient.WaitForShutdownStartAsync();
|
||||
|
||||
// Start disposing while close is still parked inside worker.ShutdownAsync.
|
||||
ValueTask disposeTask = session.DisposeAsync();
|
||||
|
||||
// Now release the worker shutdown so close can complete.
|
||||
workerClient.ReleaseShutdown();
|
||||
|
||||
// Both must complete cleanly — the close's Release() must run before the
|
||||
// dispose actually tears the semaphore down.
|
||||
SessionCloseResult result = await closeTask;
|
||||
await disposeTask;
|
||||
|
||||
Assert.Equal(SessionState.Closed, result.FinalState);
|
||||
Assert.Equal(1, workerClient.ShutdownCount);
|
||||
// Worker dispose ran exactly once even with the close/dispose interleave.
|
||||
Assert.Equal(1, workerClient.DisposeCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Double-dispose is tolerated: the second call must swallow
|
||||
/// <see cref="ObjectDisposedException"/> from the already-disposed semaphore
|
||||
/// rather than propagating it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task DisposeAsync_CalledTwice_DoesNotThrow()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
await session.CloseAsync("test-close", CancellationToken.None);
|
||||
|
||||
await session.DisposeAsync();
|
||||
// No second exception — the dispose's defensive ObjectDisposedException catch
|
||||
// covers the doubled call path that SessionManager.ShutdownAsync could trigger
|
||||
// if it re-removed a session.
|
||||
await session.DisposeAsync();
|
||||
}
|
||||
|
||||
private static GatewaySession CreateReadySession(IWorkerClient workerClient)
|
||||
{
|
||||
GatewaySession session = new(
|
||||
sessionId: "session-test",
|
||||
backendName: "mxaccess",
|
||||
pipeName: "mxaccess-gateway-1-session-test",
|
||||
nonce: "nonce",
|
||||
clientIdentity: "client-1",
|
||||
clientSessionName: "test-session",
|
||||
clientCorrelationId: "client-correlation-1",
|
||||
commandTimeout: TimeSpan.FromSeconds(5),
|
||||
startupTimeout: TimeSpan.FromSeconds(5),
|
||||
shutdownTimeout: TimeSpan.FromSeconds(5),
|
||||
leaseDuration: TimeSpan.FromMinutes(30),
|
||||
openedAt: DateTimeOffset.UtcNow);
|
||||
session.AttachWorkerClient(workerClient);
|
||||
session.MarkReady();
|
||||
return session;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal worker client that parks <see cref="ShutdownAsync"/> until the test
|
||||
/// explicitly releases it. Used to keep <see cref="GatewaySession.CloseAsync"/>
|
||||
/// stuck between its <c>Closing</c> and <c>Closed</c> writes so the test can
|
||||
/// observe and act on the intermediate state.
|
||||
/// </summary>
|
||||
private sealed class BlockingShutdownWorkerClient : IWorkerClient
|
||||
{
|
||||
private readonly TaskCompletionSource _shutdownStarted = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource _shutdownReleased = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public string SessionId { get; } = "session-test";
|
||||
|
||||
public int? ProcessId { get; } = 1234;
|
||||
|
||||
public WorkerClientState State { get; private set; } = WorkerClientState.Ready;
|
||||
|
||||
public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.UtcNow;
|
||||
|
||||
public int ShutdownCount { get; private set; }
|
||||
|
||||
public int DisposeCount { get; private set; }
|
||||
|
||||
public Task WaitForShutdownStartAsync()
|
||||
{
|
||||
return _shutdownStarted.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
public void ReleaseShutdown()
|
||||
{
|
||||
_shutdownReleased.TrySetResult();
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
WorkerCommand command,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken) => Task.FromResult(new WorkerCommandReply());
|
||||
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask.ConfigureAwait(false);
|
||||
yield break;
|
||||
}
|
||||
|
||||
public async Task ShutdownAsync(TimeSpan timeout, CancellationToken cancellationToken)
|
||||
{
|
||||
ShutdownCount++;
|
||||
_shutdownStarted.TrySetResult();
|
||||
await _shutdownReleased.Task.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
State = WorkerClientState.Closed;
|
||||
}
|
||||
|
||||
public void Kill(string reason)
|
||||
{
|
||||
State = WorkerClientState.Faulted;
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
DisposeCount++;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeWorkerClient : IWorkerClient
|
||||
{
|
||||
public string SessionId { get; } = "session-test";
|
||||
|
||||
public int? ProcessId { get; } = 1234;
|
||||
|
||||
public WorkerClientState State { get; } = WorkerClientState.Ready;
|
||||
|
||||
public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.UtcNow;
|
||||
|
||||
public int DisposeCount { get; private set; }
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
WorkerCommand command,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken) => Task.FromResult(new WorkerCommandReply());
|
||||
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask.ConfigureAwait(false);
|
||||
yield break;
|
||||
}
|
||||
|
||||
public Task ShutdownAsync(TimeSpan timeout, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public void Kill(string reason)
|
||||
{
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
DisposeCount++;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,846 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
|
||||
|
||||
/// <summary>
|
||||
/// Tests-013: per-method gateway-side coverage for every
|
||||
/// <c>GatewaySession.*BulkAsync</c> entry point. Each method gets a
|
||||
/// round-trip test that pins the <see cref="MxCommandKind"/> sent to the
|
||||
/// worker, the per-entry payload shape, a failure-mode (per-entry failure
|
||||
/// surfaced or protocol-status failure) check, and a cancellation-propagation
|
||||
/// check. The secured-write variants additionally pin that the credential
|
||||
/// payload (<c>current_user_id</c>, <c>verifier_user_id</c>) is preserved
|
||||
/// end-to-end and not flattened/redacted by the gateway's command shape.
|
||||
/// </summary>
|
||||
public sealed class SessionManagerBulkTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task AddItemBulkAsync_ForwardsOneAddItemBulkCommandAndReturnsResults()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.AddItemBulk = new BulkSubscribeReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new SubscribeResult { ServerHandle = 12, TagAddress = "Galaxy.Tag.Ok", ItemHandle = 511, WasSuccessful = true },
|
||||
new SubscribeResult { ServerHandle = 12, TagAddress = "Galaxy.Tag.Bad", ItemHandle = 0, WasSuccessful = false, ErrorMessage = "invalid tag" },
|
||||
},
|
||||
}, MxCommandKind.AddItemBulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<SubscribeResult> results = await session.AddItemBulkAsync(
|
||||
12,
|
||||
["Galaxy.Tag.Ok", "Galaxy.Tag.Bad"],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, workerClient.InvokeCount);
|
||||
Assert.Equal(MxCommandKind.AddItemBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal(12, workerClient.LastCommand?.Command.AddItemBulk.ServerHandle);
|
||||
Assert.Equal(["Galaxy.Tag.Ok", "Galaxy.Tag.Bad"], workerClient.LastCommand?.Command.AddItemBulk.TagAddresses);
|
||||
Assert.Equal(2, results.Count);
|
||||
Assert.True(results[0].WasSuccessful);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
Assert.Equal("invalid tag", results[1].ErrorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AddItemBulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.AddItemBulkAsync(12, ["Tag.A"], cts.Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AdviseItemBulkAsync_ForwardsOneAdviseItemBulkCommandAndReturnsResults()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.AdviseItemBulk = new BulkSubscribeReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new SubscribeResult { ServerHandle = 12, ItemHandle = 901, WasSuccessful = true },
|
||||
new SubscribeResult { ServerHandle = 12, ItemHandle = 902, WasSuccessful = false, ErrorMessage = "invalid item handle" },
|
||||
},
|
||||
}, MxCommandKind.AdviseItemBulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<SubscribeResult> results = await session.AdviseItemBulkAsync(
|
||||
12,
|
||||
[901, 902],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MxCommandKind.AdviseItemBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal(12, workerClient.LastCommand?.Command.AdviseItemBulk.ServerHandle);
|
||||
Assert.Equal([901, 902], workerClient.LastCommand?.Command.AdviseItemBulk.ItemHandles);
|
||||
Assert.Equal(2, results.Count);
|
||||
Assert.True(results[0].WasSuccessful);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AdviseItemBulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.AdviseItemBulkAsync(12, [101], cts.Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RemoveItemBulkAsync_ForwardsOneRemoveItemBulkCommandAndReturnsResults()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.RemoveItemBulk = new BulkSubscribeReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new SubscribeResult { ServerHandle = 12, ItemHandle = 11, WasSuccessful = true },
|
||||
new SubscribeResult { ServerHandle = 12, ItemHandle = 12, WasSuccessful = false, ErrorMessage = "unknown handle" },
|
||||
},
|
||||
}, MxCommandKind.RemoveItemBulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<SubscribeResult> results = await session.RemoveItemBulkAsync(
|
||||
12,
|
||||
[11, 12],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MxCommandKind.RemoveItemBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal([11, 12], workerClient.LastCommand?.Command.RemoveItemBulk.ItemHandles);
|
||||
Assert.Equal(2, results.Count);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RemoveItemBulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.RemoveItemBulkAsync(12, [11], cts.Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnAdviseItemBulkAsync_ForwardsOneUnAdviseItemBulkCommandAndReturnsResults()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.UnAdviseItemBulk = new BulkSubscribeReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new SubscribeResult { ServerHandle = 12, ItemHandle = 21, WasSuccessful = true },
|
||||
new SubscribeResult { ServerHandle = 12, ItemHandle = 22, WasSuccessful = false, ErrorMessage = "not advised" },
|
||||
},
|
||||
}, MxCommandKind.UnAdviseItemBulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<SubscribeResult> results = await session.UnAdviseItemBulkAsync(
|
||||
12,
|
||||
[21, 22],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MxCommandKind.UnAdviseItemBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal([21, 22], workerClient.LastCommand?.Command.UnAdviseItemBulk.ItemHandles);
|
||||
Assert.Equal(2, results.Count);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
Assert.Equal("not advised", results[1].ErrorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnAdviseItemBulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.UnAdviseItemBulkAsync(12, [21], cts.Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SubscribeBulkAsync_SurfacesPerEntryFailures()
|
||||
{
|
||||
// SubscribeBulkAsync already has a happy-path test in SessionManagerTests
|
||||
// (GatewaySessionSubscribeBulkAsync_ForwardsOneBulkCommandAndReturnsResults);
|
||||
// this complementary test pins the per-entry failure-surface behaviour.
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.SubscribeBulk = new BulkSubscribeReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new SubscribeResult { ServerHandle = 12, TagAddress = "Galaxy.Good", ItemHandle = 501, WasSuccessful = true },
|
||||
new SubscribeResult { ServerHandle = 12, TagAddress = "Galaxy.Bad", ItemHandle = 0, WasSuccessful = false, ErrorMessage = "MXAccess subscribe failed" },
|
||||
},
|
||||
}, MxCommandKind.SubscribeBulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<SubscribeResult> results = await session.SubscribeBulkAsync(
|
||||
12,
|
||||
["Galaxy.Good", "Galaxy.Bad"],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, results.Count);
|
||||
Assert.True(results[0].WasSuccessful);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
Assert.Equal("MXAccess subscribe failed", results[1].ErrorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SubscribeBulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.SubscribeBulkAsync(12, ["Tag"], cts.Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnsubscribeBulkAsync_ForwardsOneUnsubscribeBulkCommandAndReturnsResults()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.UnsubscribeBulk = new BulkSubscribeReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new SubscribeResult { ServerHandle = 12, ItemHandle = 31, WasSuccessful = true },
|
||||
new SubscribeResult { ServerHandle = 12, ItemHandle = 32, WasSuccessful = false, ErrorMessage = "unknown handle" },
|
||||
},
|
||||
}, MxCommandKind.UnsubscribeBulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<SubscribeResult> results = await session.UnsubscribeBulkAsync(
|
||||
12,
|
||||
[31, 32],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MxCommandKind.UnsubscribeBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal([31, 32], workerClient.LastCommand?.Command.UnsubscribeBulk.ItemHandles);
|
||||
Assert.Equal(2, results.Count);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnsubscribeBulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.UnsubscribeBulkAsync(12, [31], cts.Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriteBulkAsync_SurfacesPerEntryFailures()
|
||||
{
|
||||
// Complement the existing happy-path WriteBulk test in SessionManagerTests
|
||||
// with an explicit per-entry failure assertion plus payload-shape pinning.
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.WriteBulk = new BulkWriteReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkWriteResult { ServerHandle = 12, ItemHandle = 901, WasSuccessful = true },
|
||||
new BulkWriteResult { ServerHandle = 12, ItemHandle = 902, WasSuccessful = false, ErrorMessage = "MXAccess invalid handle" },
|
||||
},
|
||||
}, MxCommandKind.WriteBulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<BulkWriteResult> results = await session.WriteBulkAsync(
|
||||
12,
|
||||
new[]
|
||||
{
|
||||
new WriteBulkEntry { ItemHandle = 901, UserId = 5, Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 11 } },
|
||||
new WriteBulkEntry { ItemHandle = 902, UserId = 5, Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 22 } },
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MxCommandKind.WriteBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal(2, workerClient.LastCommand?.Command.WriteBulk.Entries.Count);
|
||||
Assert.Equal(901, workerClient.LastCommand?.Command.WriteBulk.Entries[0].ItemHandle);
|
||||
Assert.Equal(11, workerClient.LastCommand?.Command.WriteBulk.Entries[0].Value.Int32Value);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
Assert.Equal("MXAccess invalid handle", results[1].ErrorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriteBulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.WriteBulkAsync(
|
||||
12,
|
||||
new[] { new WriteBulkEntry { ItemHandle = 1, UserId = 1, Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 0 } } },
|
||||
cts.Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Write2BulkAsync_ForwardsOneWrite2BulkCommandAndPreservesTimestampPayload()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.Write2Bulk = new BulkWriteReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkWriteResult { ServerHandle = 12, ItemHandle = 701, WasSuccessful = true },
|
||||
new BulkWriteResult { ServerHandle = 12, ItemHandle = 702, WasSuccessful = false, ErrorMessage = "MXAccess Write2 failed" },
|
||||
},
|
||||
}, MxCommandKind.Write2Bulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<BulkWriteResult> results = await session.Write2BulkAsync(
|
||||
12,
|
||||
new[]
|
||||
{
|
||||
new Write2BulkEntry
|
||||
{
|
||||
ItemHandle = 701,
|
||||
UserId = 5,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 11 },
|
||||
TimestampValue = new MxValue { DataType = MxDataType.Time, Int64Value = 1234567890L },
|
||||
},
|
||||
new Write2BulkEntry
|
||||
{
|
||||
ItemHandle = 702,
|
||||
UserId = 5,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 22 },
|
||||
TimestampValue = new MxValue { DataType = MxDataType.Time, Int64Value = 1234567891L },
|
||||
},
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MxCommandKind.Write2Bulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal(12, workerClient.LastCommand?.Command.Write2Bulk.ServerHandle);
|
||||
Assert.Equal(2, workerClient.LastCommand?.Command.Write2Bulk.Entries.Count);
|
||||
Assert.Equal(701, workerClient.LastCommand?.Command.Write2Bulk.Entries[0].ItemHandle);
|
||||
Assert.Equal(1234567890L, workerClient.LastCommand?.Command.Write2Bulk.Entries[0].TimestampValue.Int64Value);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Write2BulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.Write2BulkAsync(
|
||||
12,
|
||||
new[]
|
||||
{
|
||||
new Write2BulkEntry
|
||||
{
|
||||
ItemHandle = 1,
|
||||
UserId = 1,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 0 },
|
||||
TimestampValue = new MxValue { DataType = MxDataType.Time, Int64Value = 0L },
|
||||
},
|
||||
},
|
||||
cts.Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriteSecuredBulkAsync_ForwardsOneWriteSecuredBulkCommandAndPreservesCredentialPayload()
|
||||
{
|
||||
// The secured variants carry caller credential identifiers (CurrentUserId /
|
||||
// VerifierUserId). Pin that those survive the gateway round-trip end-to-end —
|
||||
// the over-the-wire command shape must NOT redact or flatten them, only the
|
||||
// *log surface* (see GatewaySession's redaction rules) is allowed to drop them.
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.WriteSecuredBulk = new BulkWriteReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkWriteResult { ServerHandle = 12, ItemHandle = 601, WasSuccessful = true },
|
||||
new BulkWriteResult { ServerHandle = 12, ItemHandle = 602, WasSuccessful = false, ErrorMessage = "MXAccess secured-write rejected" },
|
||||
},
|
||||
}, MxCommandKind.WriteSecuredBulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<BulkWriteResult> results = await session.WriteSecuredBulkAsync(
|
||||
12,
|
||||
new[]
|
||||
{
|
||||
new WriteSecuredBulkEntry
|
||||
{
|
||||
ItemHandle = 601,
|
||||
CurrentUserId = 7,
|
||||
VerifierUserId = 8,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 1 },
|
||||
},
|
||||
new WriteSecuredBulkEntry
|
||||
{
|
||||
ItemHandle = 602,
|
||||
CurrentUserId = 7,
|
||||
VerifierUserId = 8,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 2 },
|
||||
},
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MxCommandKind.WriteSecuredBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal(12, workerClient.LastCommand?.Command.WriteSecuredBulk.ServerHandle);
|
||||
Assert.Equal(2, workerClient.LastCommand?.Command.WriteSecuredBulk.Entries.Count);
|
||||
WriteSecuredBulkEntry firstEntry = workerClient.LastCommand!.Command.WriteSecuredBulk.Entries[0];
|
||||
Assert.Equal(601, firstEntry.ItemHandle);
|
||||
Assert.Equal(7, firstEntry.CurrentUserId);
|
||||
Assert.Equal(8, firstEntry.VerifierUserId);
|
||||
Assert.Equal(1, firstEntry.Value.Int32Value);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
Assert.Equal("MXAccess secured-write rejected", results[1].ErrorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriteSecuredBulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.WriteSecuredBulkAsync(
|
||||
12,
|
||||
new[]
|
||||
{
|
||||
new WriteSecuredBulkEntry
|
||||
{
|
||||
ItemHandle = 1,
|
||||
CurrentUserId = 7,
|
||||
VerifierUserId = 8,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 0 },
|
||||
},
|
||||
},
|
||||
cts.Token));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests-022: Pin mid-flight cancellation behaviour for at least one bulk
|
||||
/// path. Unlike the pre-cancel <c>WriteSecuredBulkAsync_PropagatesCancellation</c>
|
||||
/// above, this fake's <see cref="MidFlightBulkWorkerClient.InvokeAsync"/>
|
||||
/// returns a <see cref="TaskCompletionSource"/>-backed task that does NOT
|
||||
/// complete until the registered token fires. The session call therefore
|
||||
/// reaches <c>InvokeBulkInternalAsync</c> → <c>InvokeAsync</c> →
|
||||
/// <c>workerClient.InvokeAsync</c> and parks on an in-flight await; only
|
||||
/// after that does <c>cts.CancelAsync()</c> fire. This is the path a real
|
||||
/// client closing its stream would hit, which the pre-cancel pattern can't
|
||||
/// exercise.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task WriteSecuredBulkAsync_WhenCancelledMidFlight_ThrowsOperationCanceledForRequestToken()
|
||||
{
|
||||
MidFlightBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
|
||||
Task<IReadOnlyList<BulkWriteResult>> writeTask = session.WriteSecuredBulkAsync(
|
||||
12,
|
||||
new[]
|
||||
{
|
||||
new WriteSecuredBulkEntry
|
||||
{
|
||||
ItemHandle = 1,
|
||||
CurrentUserId = 7,
|
||||
VerifierUserId = 8,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 0 },
|
||||
},
|
||||
},
|
||||
cts.Token);
|
||||
|
||||
// Wait until the gateway has descended into the worker's InvokeAsync and
|
||||
// registered its cancellation continuation — only then is this a true
|
||||
// mid-flight cancel.
|
||||
await workerClient.InvokeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
Assert.False(writeTask.IsCompleted);
|
||||
|
||||
await cts.CancelAsync();
|
||||
|
||||
OperationCanceledException exception = await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await writeTask);
|
||||
Assert.Equal(cts.Token, exception.CancellationToken);
|
||||
Assert.Equal(1, workerClient.InvokeCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriteSecured2BulkAsync_ForwardsOneWriteSecured2BulkCommandAndPreservesCredentialAndTimestampPayload()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.WriteSecured2Bulk = new BulkWriteReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkWriteResult { ServerHandle = 12, ItemHandle = 801, WasSuccessful = true },
|
||||
new BulkWriteResult { ServerHandle = 12, ItemHandle = 802, WasSuccessful = false, ErrorMessage = "MXAccess secured2-write rejected" },
|
||||
},
|
||||
}, MxCommandKind.WriteSecured2Bulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<BulkWriteResult> results = await session.WriteSecured2BulkAsync(
|
||||
12,
|
||||
new[]
|
||||
{
|
||||
new WriteSecured2BulkEntry
|
||||
{
|
||||
ItemHandle = 801,
|
||||
CurrentUserId = 7,
|
||||
VerifierUserId = 8,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 1 },
|
||||
TimestampValue = new MxValue { DataType = MxDataType.Time, Int64Value = 1700000000L },
|
||||
},
|
||||
new WriteSecured2BulkEntry
|
||||
{
|
||||
ItemHandle = 802,
|
||||
CurrentUserId = 7,
|
||||
VerifierUserId = 8,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 2 },
|
||||
TimestampValue = new MxValue { DataType = MxDataType.Time, Int64Value = 1700000001L },
|
||||
},
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MxCommandKind.WriteSecured2Bulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal(2, workerClient.LastCommand?.Command.WriteSecured2Bulk.Entries.Count);
|
||||
WriteSecured2BulkEntry firstEntry = workerClient.LastCommand!.Command.WriteSecured2Bulk.Entries[0];
|
||||
Assert.Equal(801, firstEntry.ItemHandle);
|
||||
Assert.Equal(7, firstEntry.CurrentUserId);
|
||||
Assert.Equal(8, firstEntry.VerifierUserId);
|
||||
Assert.Equal(1, firstEntry.Value.Int32Value);
|
||||
Assert.Equal(1700000000L, firstEntry.TimestampValue.Int64Value);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriteSecured2BulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.WriteSecured2BulkAsync(
|
||||
12,
|
||||
new[]
|
||||
{
|
||||
new WriteSecured2BulkEntry
|
||||
{
|
||||
ItemHandle = 1,
|
||||
CurrentUserId = 7,
|
||||
VerifierUserId = 8,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 0 },
|
||||
TimestampValue = new MxValue { DataType = MxDataType.Time, Int64Value = 0L },
|
||||
},
|
||||
},
|
||||
cts.Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadBulkAsync_SurfacesPerEntryFailures()
|
||||
{
|
||||
// Complement the existing happy-path ReadBulk test in SessionManagerTests
|
||||
// with the failure-mode case where one tag failed to read.
|
||||
FakeBulkWorkerClient workerClient = WithReply(reply => reply.ReadBulk = new BulkReadReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkReadResult
|
||||
{
|
||||
ServerHandle = 12,
|
||||
TagAddress = "Galaxy.Good",
|
||||
ItemHandle = 511,
|
||||
WasSuccessful = true,
|
||||
WasCached = false,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 42 },
|
||||
},
|
||||
new BulkReadResult
|
||||
{
|
||||
ServerHandle = 12,
|
||||
TagAddress = "Galaxy.Bad",
|
||||
ItemHandle = 0,
|
||||
WasSuccessful = false,
|
||||
ErrorMessage = "MXAccess read timed out",
|
||||
},
|
||||
},
|
||||
}, MxCommandKind.ReadBulk);
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
|
||||
IReadOnlyList<BulkReadResult> results = await session.ReadBulkAsync(
|
||||
12,
|
||||
["Galaxy.Good", "Galaxy.Bad"],
|
||||
TimeSpan.FromMilliseconds(750),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MxCommandKind.ReadBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal(750u, workerClient.LastCommand?.Command.ReadBulk.TimeoutMs);
|
||||
Assert.Equal(["Galaxy.Good", "Galaxy.Bad"], workerClient.LastCommand?.Command.ReadBulk.TagAddresses);
|
||||
Assert.Equal(2, results.Count);
|
||||
Assert.True(results[0].WasSuccessful);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
Assert.Equal("MXAccess read timed out", results[1].ErrorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadBulkAsync_PropagatesCancellation()
|
||||
{
|
||||
FakeBulkWorkerClient workerClient = new();
|
||||
GatewaySession session = await OpenSessionAsync(workerClient);
|
||||
using CancellationTokenSource cts = new();
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await session.ReadBulkAsync(
|
||||
12,
|
||||
["Galaxy.Tag"],
|
||||
TimeSpan.FromMilliseconds(500),
|
||||
cts.Token));
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Helpers
|
||||
// -----------------------------------------------------------------
|
||||
|
||||
private static FakeBulkWorkerClient WithReply(Action<MxCommandReply> populate, MxCommandKind kind)
|
||||
{
|
||||
MxCommandReply reply = new()
|
||||
{
|
||||
SessionId = "session-1",
|
||||
CorrelationId = "correlation-1",
|
||||
Kind = kind,
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
};
|
||||
populate(reply);
|
||||
return new FakeBulkWorkerClient
|
||||
{
|
||||
InvokeReply = new WorkerCommandReply { Reply = reply },
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<GatewaySession> OpenSessionAsync(FakeBulkWorkerClient workerClient)
|
||||
{
|
||||
return await OpenSessionAsync((IWorkerClient)workerClient);
|
||||
}
|
||||
|
||||
private static async Task<GatewaySession> OpenSessionAsync(IWorkerClient workerClient)
|
||||
{
|
||||
SessionManager manager = CreateManager(workerClient);
|
||||
return await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
}
|
||||
|
||||
private static SessionManager CreateManager(IWorkerClient workerClient)
|
||||
{
|
||||
return new SessionManager(
|
||||
new SessionRegistry(),
|
||||
new FakeBulkSessionWorkerClientFactory(workerClient),
|
||||
Options.Create(new GatewayOptions
|
||||
{
|
||||
Sessions = new SessionOptions
|
||||
{
|
||||
DefaultCommandTimeoutSeconds = 30,
|
||||
MaxSessions = 16,
|
||||
DefaultLeaseSeconds = 1800,
|
||||
},
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
StartupTimeoutSeconds = 30,
|
||||
ShutdownTimeoutSeconds = 10,
|
||||
},
|
||||
}),
|
||||
new GatewayMetrics());
|
||||
}
|
||||
|
||||
private static SessionOpenRequest CreateOpenRequest()
|
||||
{
|
||||
return new SessionOpenRequest(
|
||||
RequestedBackend: null,
|
||||
ClientSessionName: "test-session",
|
||||
ClientCorrelationId: "client-correlation-1",
|
||||
CommandTimeout: Duration.FromTimeSpan(TimeSpan.FromSeconds(5)));
|
||||
}
|
||||
|
||||
private sealed class FakeBulkSessionWorkerClientFactory(IWorkerClient workerClient) : ISessionWorkerClientFactory
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<IWorkerClient> CreateAsync(
|
||||
GatewaySession session,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(workerClient);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeBulkWorkerClient : IWorkerClient
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public string SessionId { get; init; } = "session-1";
|
||||
|
||||
/// <inheritdoc />
|
||||
public int? ProcessId { get; init; } = 1234;
|
||||
|
||||
/// <inheritdoc />
|
||||
public WorkerClientState State { get; set; } = WorkerClientState.Ready;
|
||||
|
||||
/// <inheritdoc />
|
||||
public DateTimeOffset LastHeartbeatAt { get; init; } = DateTimeOffset.UtcNow;
|
||||
|
||||
/// <summary>Gets the number of times Invoke was called on the fake worker client.</summary>
|
||||
public int InvokeCount { get; private set; }
|
||||
|
||||
/// <summary>Gets the last command invoked on the fake worker client.</summary>
|
||||
public WorkerCommand? LastCommand { get; private set; }
|
||||
|
||||
/// <summary>Gets the reply to return for invoke calls on the fake worker client.</summary>
|
||||
public WorkerCommandReply? InvokeReply { get; init; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
WorkerCommand command,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
InvokeCount++;
|
||||
LastCommand = command;
|
||||
if (InvokeReply is not null)
|
||||
{
|
||||
return Task.FromResult(InvokeReply);
|
||||
}
|
||||
|
||||
MxCommandKind kind = command.Command?.Kind ?? MxCommandKind.Unspecified;
|
||||
return Task.FromResult(new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
CorrelationId = "correlation-1",
|
||||
Kind = kind,
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ShutdownAsync(TimeSpan timeout, CancellationToken cancellationToken)
|
||||
{
|
||||
State = WorkerClientState.Closed;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Kill(string reason) => State = WorkerClientState.Faulted;
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mid-flight cancellation fake for Tests-022.
|
||||
/// <see cref="InvokeAsync"/> signals <see cref="InvokeStarted"/>, registers
|
||||
/// a cancellation continuation on the caller's <see cref="CancellationToken"/>,
|
||||
/// and parks on a <see cref="TaskCompletionSource{TResult}"/> that completes
|
||||
/// only when the token fires or the fake is shut down. This is the only
|
||||
/// way to land an <see cref="OperationCanceledException"/> on the async
|
||||
/// continuation rather than the synchronous fast-path inside
|
||||
/// <c>ThrowIfCancellationRequested</c>.
|
||||
/// </summary>
|
||||
private sealed class MidFlightBulkWorkerClient : IWorkerClient
|
||||
{
|
||||
private readonly TaskCompletionSource<WorkerCommandReply> _invokeCompletion =
|
||||
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
/// <inheritdoc />
|
||||
public string SessionId { get; init; } = "session-1";
|
||||
|
||||
/// <inheritdoc />
|
||||
public int? ProcessId { get; init; } = 1234;
|
||||
|
||||
/// <inheritdoc />
|
||||
public WorkerClientState State { get; set; } = WorkerClientState.Ready;
|
||||
|
||||
/// <inheritdoc />
|
||||
public DateTimeOffset LastHeartbeatAt { get; init; } = DateTimeOffset.UtcNow;
|
||||
|
||||
/// <summary>Gets the number of times <see cref="InvokeAsync"/> was entered.</summary>
|
||||
public int InvokeCount { get; private set; }
|
||||
|
||||
/// <summary>Signals when <see cref="InvokeAsync"/> first enters — the test
|
||||
/// awaits this before triggering mid-flight cancellation.</summary>
|
||||
public TaskCompletionSource InvokeStarted { get; } =
|
||||
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
WorkerCommand command,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
InvokeCount++;
|
||||
// Register cancellation BEFORE signalling start so the test can be
|
||||
// certain the continuation is wired the moment InvokeStarted resolves.
|
||||
cancellationToken.Register(() => _invokeCompletion.TrySetCanceled(cancellationToken));
|
||||
InvokeStarted.TrySetResult();
|
||||
return _invokeCompletion.Task;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ShutdownAsync(TimeSpan timeout, CancellationToken cancellationToken)
|
||||
{
|
||||
State = WorkerClientState.Closed;
|
||||
_invokeCompletion.TrySetCanceled(cancellationToken);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Kill(string reason)
|
||||
{
|
||||
State = WorkerClientState.Faulted;
|
||||
_invokeCompletion.TrySetCanceled();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
_invokeCompletion.TrySetCanceled();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,797 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
|
||||
|
||||
public sealed class SessionManagerTests
|
||||
{
|
||||
/// <summary>Verifies that opening a session with a ready worker registers the session in ready state.</summary>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_WithWorkerReady_RegistersReadySession()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
FakeSessionWorkerClientFactory factory = new(workerClient)
|
||||
{
|
||||
ApplyLifecycleTransitions = true,
|
||||
};
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(factory, metrics: metrics);
|
||||
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
|
||||
Assert.True(manager.TryGetSession(session.SessionId, out GatewaySession? registered));
|
||||
Assert.Same(session, registered);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
Assert.Equal("client-1", session.ClientIdentity);
|
||||
Assert.Equal(["StartingWorker", "WaitingForPipe", "Handshaking", "InitializingWorker"], factory.ObservedStates);
|
||||
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
|
||||
Assert.Equal(1, metrics.GetSnapshot().SessionsOpened);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that opening a session sets the initial lease expiry from the configured default lease.</summary>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_SetsInitialDefaultLease()
|
||||
{
|
||||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-04-29T10:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||
GatewayOptions options = CreateOptions(defaultLeaseSeconds: 1800);
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(new FakeWorkerClient()),
|
||||
options: options,
|
||||
timeProvider: clock);
|
||||
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
|
||||
Assert.Equal(clock.GetUtcNow() + TimeSpan.FromMinutes(30), session.LeaseExpiresAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_GeneratesClientCorrelationIdFromClientNameAndSessionId()
|
||||
{
|
||||
SessionOpenRequest request = CreateOpenRequest() with
|
||||
{
|
||||
ClientSessionName = "rust-load-client",
|
||||
ClientCorrelationId = "caller-provided-correlation",
|
||||
};
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(new FakeWorkerClient()));
|
||||
|
||||
GatewaySession session = await manager.OpenSessionAsync(request, "client-1", CancellationToken.None);
|
||||
|
||||
Assert.Equal($"rust-load-client-{session.SessionId}", session.ClientCorrelationId);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that opening a session without a client session name uses the client correlation prefix.</summary>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_WhenClientSessionNameMissing_UsesClientCorrelationPrefix()
|
||||
{
|
||||
SessionOpenRequest request = CreateOpenRequest() with
|
||||
{
|
||||
ClientSessionName = "",
|
||||
};
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(new FakeWorkerClient()));
|
||||
|
||||
GatewaySession session = await manager.OpenSessionAsync(request, "client-1", CancellationToken.None);
|
||||
|
||||
Assert.Equal($"client-{session.SessionId}", session.ClientCorrelationId);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that invoking a command on a ready session forwards the command to the worker.</summary>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WhenSessionReady_ForwardsCommandToWorker()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
|
||||
WorkerCommandReply reply = await manager.InvokeAsync(
|
||||
session.SessionId,
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, workerClient.InvokeCount);
|
||||
Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that invoking a command on a ready session refreshes its lease expiry.</summary>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WhenSessionReady_RefreshesLease()
|
||||
{
|
||||
GatewaySession session = new(
|
||||
"session-lease-refresh",
|
||||
"mxaccess",
|
||||
"mxaccess-gateway-1-session-lease-refresh",
|
||||
"nonce",
|
||||
"client-1",
|
||||
"test-session",
|
||||
"client-correlation-1",
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromSeconds(5),
|
||||
TimeSpan.FromSeconds(5),
|
||||
TimeSpan.FromMinutes(30),
|
||||
DateTimeOffset.UtcNow - TimeSpan.FromHours(1));
|
||||
session.AttachWorkerClient(new FakeWorkerClient());
|
||||
session.MarkReady();
|
||||
DateTimeOffset? initialLease = session.LeaseExpiresAt;
|
||||
|
||||
await session.InvokeAsync(CreateCommand(MxCommandKind.Ping), CancellationToken.None);
|
||||
|
||||
Assert.True(session.LeaseExpiresAt > initialLease);
|
||||
Assert.True(session.LeaseExpiresAt > DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GatewaySessionSubscribeBulkAsync_ForwardsOneBulkCommandAndReturnsResults()
|
||||
{
|
||||
FakeWorkerClient workerClient = new()
|
||||
{
|
||||
InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = "session-1",
|
||||
CorrelationId = "correlation-1",
|
||||
Kind = MxCommandKind.SubscribeBulk,
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
SubscribeBulk = new BulkSubscribeReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new SubscribeResult
|
||||
{
|
||||
ServerHandle = 12,
|
||||
TagAddress = "Galaxy.Tag.Value",
|
||||
ItemHandle = 512,
|
||||
WasSuccessful = true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
|
||||
IReadOnlyList<SubscribeResult> results = await session.SubscribeBulkAsync(
|
||||
12,
|
||||
["Galaxy.Tag.Value"],
|
||||
CancellationToken.None);
|
||||
|
||||
SubscribeResult result = Assert.Single(results);
|
||||
Assert.Equal(512, result.ItemHandle);
|
||||
Assert.Equal(1, workerClient.InvokeCount);
|
||||
Assert.Equal(MxCommandKind.SubscribeBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal(["Galaxy.Tag.Value"], workerClient.LastCommand?.Command.SubscribeBulk.TagAddresses);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GatewaySessionWriteBulkAsync_ForwardsOneBulkCommandAndReturnsResults()
|
||||
{
|
||||
FakeWorkerClient workerClient = new()
|
||||
{
|
||||
InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = "session-1",
|
||||
CorrelationId = "correlation-1",
|
||||
Kind = MxCommandKind.WriteBulk,
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
WriteBulk = new BulkWriteReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkWriteResult
|
||||
{
|
||||
ServerHandle = 12,
|
||||
ItemHandle = 901,
|
||||
WasSuccessful = true,
|
||||
},
|
||||
new BulkWriteResult
|
||||
{
|
||||
ServerHandle = 12,
|
||||
ItemHandle = 902,
|
||||
WasSuccessful = false,
|
||||
ErrorMessage = "MXAccess invalid handle",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
|
||||
IReadOnlyList<BulkWriteResult> results = await session.WriteBulkAsync(
|
||||
12,
|
||||
new[]
|
||||
{
|
||||
new WriteBulkEntry
|
||||
{
|
||||
ItemHandle = 901,
|
||||
UserId = 5,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 11 },
|
||||
},
|
||||
new WriteBulkEntry
|
||||
{
|
||||
ItemHandle = 902,
|
||||
UserId = 5,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 22 },
|
||||
},
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, results.Count);
|
||||
Assert.True(results[0].WasSuccessful);
|
||||
Assert.False(results[1].WasSuccessful);
|
||||
Assert.Equal(MxCommandKind.WriteBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal(2, workerClient.LastCommand?.Command.WriteBulk.Entries.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GatewaySessionReadBulkAsync_ForwardsOneBulkCommandAndReturnsResults()
|
||||
{
|
||||
FakeWorkerClient workerClient = new()
|
||||
{
|
||||
InvokeReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = "session-1",
|
||||
CorrelationId = "correlation-1",
|
||||
Kind = MxCommandKind.ReadBulk,
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
ReadBulk = new BulkReadReply
|
||||
{
|
||||
Results =
|
||||
{
|
||||
new BulkReadResult
|
||||
{
|
||||
ServerHandle = 12,
|
||||
TagAddress = "Galaxy.Tag.Value",
|
||||
ItemHandle = 512,
|
||||
WasSuccessful = true,
|
||||
WasCached = true,
|
||||
Value = new MxValue { DataType = MxDataType.Integer, Int32Value = 42 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
|
||||
IReadOnlyList<BulkReadResult> results = await session.ReadBulkAsync(
|
||||
12,
|
||||
["Galaxy.Tag.Value"],
|
||||
TimeSpan.FromMilliseconds(500),
|
||||
CancellationToken.None);
|
||||
|
||||
BulkReadResult result = Assert.Single(results);
|
||||
Assert.True(result.WasSuccessful);
|
||||
Assert.True(result.WasCached);
|
||||
Assert.Equal(42, result.Value.Int32Value);
|
||||
Assert.Equal(MxCommandKind.ReadBulk, workerClient.LastCommand?.Command.Kind);
|
||||
Assert.Equal(["Galaxy.Tag.Value"], workerClient.LastCommand?.Command.ReadBulk.TagAddresses);
|
||||
Assert.Equal(500u, workerClient.LastCommand?.Command.ReadBulk.TimeoutMs);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that invoking a command on a faulted session rejects the command.</summary>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WhenSessionFaulted_RejectsCommand()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
session.MarkFaulted("test fault");
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.InvokeAsync(
|
||||
session.SessionId,
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
CancellationToken.None));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, exception.ErrorCode);
|
||||
Assert.Equal(0, workerClient.InvokeCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Server-030 regression: when the gateway-side <c>SessionState</c> is
|
||||
/// <c>Ready</c> but the worker client's own state is not, the diagnostic
|
||||
/// must surface both states so the mismatch is actionable instead of
|
||||
/// producing a self-contradictory "Session ... is not ready. Current
|
||||
/// state is Ready." message.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WhenWorkerNotReadyButSessionReady_DiagnosticIncludesBothStates()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
|
||||
// Force a state mismatch: session stays Ready, worker transitions out.
|
||||
workerClient.State = WorkerClientState.Handshaking;
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.InvokeAsync(
|
||||
session.SessionId,
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
CancellationToken.None));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, exception.ErrorCode);
|
||||
Assert.Contains("Session state is Ready", exception.Message);
|
||||
Assert.Contains("worker state is Handshaking", exception.Message);
|
||||
Assert.Equal(0, workerClient.InvokeCount);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that closing a session removes it from the registry.</summary>
|
||||
[Fact]
|
||||
public async Task CloseSessionAsync_RemovesClosedSession()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), metrics: metrics);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
|
||||
SessionCloseResult firstClose = await manager.CloseSessionAsync(session.SessionId, CancellationToken.None);
|
||||
SessionManagerException secondClose = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.CloseSessionAsync(session.SessionId, CancellationToken.None));
|
||||
|
||||
Assert.False(firstClose.AlreadyClosed);
|
||||
Assert.Equal(SessionState.Closed, firstClose.FinalState);
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotFound, secondClose.ErrorCode);
|
||||
Assert.Equal(1, workerClient.ShutdownCount);
|
||||
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
|
||||
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that closing a session kills the worker when shutdown fails.</summary>
|
||||
[Fact]
|
||||
public async Task CloseSessionAsync_WhenWorkerShutdownFails_KillsWorker()
|
||||
{
|
||||
FakeWorkerClient workerClient = new()
|
||||
{
|
||||
ShutdownException = new WorkerClientException(
|
||||
WorkerClientErrorCode.ShutdownTimeout,
|
||||
"Worker shutdown timed out."),
|
||||
};
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.CloseSessionAsync(session.SessionId, CancellationToken.None));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
|
||||
Assert.Equal(1, workerClient.ShutdownCount);
|
||||
Assert.Equal(1, workerClient.KillCount);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that when worker shutdown fails, the session is removed and the slot is released.</summary>
|
||||
[Fact]
|
||||
public async Task CloseSessionAsync_WhenWorkerShutdownFails_RemovesSessionAndReleasesSlot()
|
||||
{
|
||||
FakeWorkerClient failingWorkerClient = new()
|
||||
{
|
||||
ShutdownException = new WorkerClientException(
|
||||
WorkerClientErrorCode.ShutdownTimeout,
|
||||
"Worker shutdown timed out."),
|
||||
};
|
||||
FakeWorkerClient replacementWorkerClient = new();
|
||||
SessionRegistry registry = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(
|
||||
new QueueingSessionWorkerClientFactory(failingWorkerClient, replacementWorkerClient),
|
||||
registry,
|
||||
metrics,
|
||||
CreateOptions(maxSessions: 1));
|
||||
GatewaySession firstSession = await manager.OpenSessionAsync(
|
||||
CreateOpenRequest(),
|
||||
"client-1",
|
||||
CancellationToken.None);
|
||||
metrics.EventReceived(firstSession.SessionId, MxEventFamily.OnDataChange.ToString());
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.CloseSessionAsync(firstSession.SessionId, CancellationToken.None));
|
||||
GatewaySession secondSession = await manager.OpenSessionAsync(
|
||||
CreateOpenRequest(),
|
||||
"client-2",
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
|
||||
Assert.False(manager.TryGetSession(firstSession.SessionId, out _));
|
||||
Assert.True(manager.TryGetSession(secondSession.SessionId, out _));
|
||||
Assert.Equal(1, registry.Count);
|
||||
Assert.Equal(1, failingWorkerClient.KillCount);
|
||||
Assert.Equal(1, failingWorkerClient.DisposeCount);
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(0, snapshot.SessionsClosed);
|
||||
Assert.False(snapshot.EventsBySession.ContainsKey(firstSession.SessionId));
|
||||
Assert.Equal(1, snapshot.OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that when the second close is canceled, the session is not removed if owned by the first close.</summary>
|
||||
[Fact]
|
||||
public async Task CloseSessionAsync_WhenSecondCloseIsCanceled_DoesNotRemoveSessionOwnedByFirstClose()
|
||||
{
|
||||
FakeWorkerClient workerClient = new()
|
||||
{
|
||||
BlockShutdown = true,
|
||||
};
|
||||
SessionRegistry registry = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
registry,
|
||||
metrics,
|
||||
CreateOptions(maxSessions: 1));
|
||||
GatewaySession session = await manager.OpenSessionAsync(
|
||||
CreateOpenRequest(),
|
||||
"client-1",
|
||||
CancellationToken.None);
|
||||
|
||||
Task<SessionCloseResult> firstClose = manager.CloseSessionAsync(session.SessionId, CancellationToken.None);
|
||||
await workerClient.WaitForShutdownStartAsync();
|
||||
using CancellationTokenSource secondCloseCancellation = new();
|
||||
Task<SessionCloseResult> secondClose = manager.CloseSessionAsync(
|
||||
session.SessionId,
|
||||
secondCloseCancellation.Token);
|
||||
|
||||
await secondCloseCancellation.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
async () => await secondClose);
|
||||
Assert.True(manager.TryGetSession(session.SessionId, out _));
|
||||
Assert.Equal(1, registry.Count);
|
||||
Assert.Equal(0, workerClient.DisposeCount);
|
||||
Assert.Equal(0, metrics.GetSnapshot().SessionsClosed);
|
||||
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
|
||||
|
||||
workerClient.ReleaseShutdown();
|
||||
SessionCloseResult closeResult = await firstClose;
|
||||
|
||||
Assert.Equal(SessionState.Closed, closeResult.FinalState);
|
||||
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
||||
Assert.Equal(0, registry.Count);
|
||||
Assert.Equal(1, workerClient.DisposeCount);
|
||||
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
|
||||
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that when worker creation fails, the session is removed from the registry.</summary>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_WhenWorkerCreationFails_RemovesSessionFromRegistry()
|
||||
{
|
||||
SessionRegistry registry = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(
|
||||
new FailingSessionWorkerClientFactory(),
|
||||
registry,
|
||||
metrics);
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
|
||||
Assert.Equal(0, registry.Count);
|
||||
Assert.Equal(0, metrics.GetSnapshot().SessionsOpened);
|
||||
Assert.Equal(1, metrics.GetSnapshot().Faults);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that closing expired leases only closes expired sessions.</summary>
|
||||
[Fact]
|
||||
public async Task CloseExpiredLeasesAsync_ClosesExpiredSessionsOnly()
|
||||
{
|
||||
FakeWorkerClient expiredClient = new();
|
||||
FakeWorkerClient activeClient = new();
|
||||
QueueingSessionWorkerClientFactory factory = new(expiredClient, activeClient);
|
||||
SessionManager manager = CreateManager(factory);
|
||||
GatewaySession expiredSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
GatewaySession activeSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", CancellationToken.None);
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
expiredSession.ExtendLease(now.AddSeconds(-1));
|
||||
activeSession.ExtendLease(now.AddMinutes(5));
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, closedCount);
|
||||
Assert.Equal(SessionState.Closed, expiredSession.State);
|
||||
Assert.Equal(SessionState.Ready, activeSession.State);
|
||||
Assert.Equal(1, expiredClient.ShutdownCount);
|
||||
Assert.Equal(0, activeClient.ShutdownCount);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that an expired-lease sweep leaves a session with an active event subscriber open.</summary>
|
||||
[Fact]
|
||||
public async Task CloseExpiredLeasesAsync_DoesNotCloseActiveEventSubscriber()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
session.ExtendLease(now.AddSeconds(-1));
|
||||
using IDisposable eventSubscriber = session.AttachEventSubscriber(allowMultipleSubscribers: false);
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, closedCount);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
Assert.Equal(0, workerClient.ShutdownCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_ClosesAllRegisteredSessions()
|
||||
{
|
||||
FakeWorkerClient firstClient = new();
|
||||
FakeWorkerClient secondClient = new();
|
||||
QueueingSessionWorkerClientFactory factory = new(firstClient, secondClient);
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(factory, metrics: metrics);
|
||||
GatewaySession firstSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", CancellationToken.None);
|
||||
GatewaySession secondSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", CancellationToken.None);
|
||||
|
||||
await manager.ShutdownAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(SessionState.Closed, firstSession.State);
|
||||
Assert.Equal(SessionState.Closed, secondSession.State);
|
||||
Assert.Equal(1, firstClient.ShutdownCount);
|
||||
Assert.Equal(1, secondClient.ShutdownCount);
|
||||
Assert.Equal(2, metrics.GetSnapshot().SessionsClosed);
|
||||
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>Creates a session manager for testing.</summary>
|
||||
/// <param name="factory">Worker client factory.</param>
|
||||
/// <param name="registry">Session registry; defaults to a new registry.</param>
|
||||
/// <param name="metrics">Metrics collector; defaults to a new instance.</param>
|
||||
/// <param name="options">Gateway options; defaults to test defaults.</param>
|
||||
/// <returns>Configured session manager.</returns>
|
||||
private static SessionManager CreateManager(
|
||||
ISessionWorkerClientFactory factory,
|
||||
ISessionRegistry? registry = null,
|
||||
GatewayMetrics? metrics = null,
|
||||
GatewayOptions? options = null,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
return new SessionManager(
|
||||
registry ?? new SessionRegistry(),
|
||||
factory,
|
||||
Options.Create(options ?? CreateOptions()),
|
||||
metrics ?? new GatewayMetrics(),
|
||||
timeProvider);
|
||||
}
|
||||
|
||||
private static GatewayOptions CreateOptions(
|
||||
int maxSessions = 64,
|
||||
int defaultLeaseSeconds = 1800)
|
||||
{
|
||||
return new GatewayOptions
|
||||
{
|
||||
Sessions = new SessionOptions
|
||||
{
|
||||
DefaultCommandTimeoutSeconds = 30,
|
||||
MaxSessions = maxSessions,
|
||||
DefaultLeaseSeconds = defaultLeaseSeconds,
|
||||
},
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
StartupTimeoutSeconds = 30,
|
||||
ShutdownTimeoutSeconds = 10,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static SessionOpenRequest CreateOpenRequest()
|
||||
{
|
||||
return new SessionOpenRequest(
|
||||
RequestedBackend: null,
|
||||
ClientSessionName: "test-session",
|
||||
ClientCorrelationId: "client-correlation-1",
|
||||
CommandTimeout: Duration.FromTimeSpan(TimeSpan.FromSeconds(5)));
|
||||
}
|
||||
|
||||
private static WorkerCommand CreateCommand(MxCommandKind kind)
|
||||
{
|
||||
return new WorkerCommand
|
||||
{
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = kind,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class FakeSessionWorkerClientFactory(IWorkerClient workerClient) : ISessionWorkerClientFactory
|
||||
{
|
||||
/// <summary>Gets the list of observed session states during worker creation.</summary>
|
||||
public List<string> ObservedStates { get; } = [];
|
||||
|
||||
/// <summary>Gets or sets a value indicating whether to apply lifecycle transitions during worker creation.</summary>
|
||||
public bool ApplyLifecycleTransitions { get; init; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IWorkerClient> CreateAsync(
|
||||
GatewaySession session,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ObservedStates.Add(session.State.ToString());
|
||||
if (ApplyLifecycleTransitions)
|
||||
{
|
||||
session.TransitionTo(SessionState.WaitingForPipe);
|
||||
ObservedStates.Add(session.State.ToString());
|
||||
session.TransitionTo(SessionState.Handshaking);
|
||||
ObservedStates.Add(session.State.ToString());
|
||||
session.TransitionTo(SessionState.InitializingWorker);
|
||||
ObservedStates.Add(session.State.ToString());
|
||||
}
|
||||
|
||||
return Task.FromResult(workerClient);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class QueueingSessionWorkerClientFactory : ISessionWorkerClientFactory
|
||||
{
|
||||
private readonly Queue<IWorkerClient> _workerClients;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="QueueingSessionWorkerClientFactory"/> class.</summary>
|
||||
/// <param name="workerClients">Array of worker clients to queue.</param>
|
||||
public QueueingSessionWorkerClientFactory(params IWorkerClient[] workerClients)
|
||||
{
|
||||
_workerClients = new Queue<IWorkerClient>(workerClients);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IWorkerClient> CreateAsync(
|
||||
GatewaySession session,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(_workerClients.Dequeue());
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FailingSessionWorkerClientFactory : ISessionWorkerClientFactory
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<IWorkerClient> CreateAsync(
|
||||
GatewaySession session,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
throw new InvalidOperationException("worker startup failed");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeWorkerClient : IWorkerClient
|
||||
{
|
||||
/// <summary>Gets the session ID for the fake worker client.</summary>
|
||||
public string SessionId { get; init; } = "session-1";
|
||||
|
||||
/// <summary>Gets the process ID for the fake worker client.</summary>
|
||||
public int? ProcessId { get; init; } = 1234;
|
||||
|
||||
/// <summary>Gets or sets the state of the fake worker client.</summary>
|
||||
public WorkerClientState State { get; set; } = WorkerClientState.Ready;
|
||||
|
||||
/// <summary>Gets the last heartbeat timestamp for the fake worker client.</summary>
|
||||
public DateTimeOffset LastHeartbeatAt { get; init; } = DateTimeOffset.UtcNow;
|
||||
|
||||
/// <summary>Gets the number of times invoke was called on the fake worker client.</summary>
|
||||
public int InvokeCount { get; private set; }
|
||||
|
||||
/// <summary>Gets the number of times shutdown was called on the fake worker client.</summary>
|
||||
public int ShutdownCount { get; private set; }
|
||||
|
||||
/// <summary>Gets the number of times kill was called on the fake worker client.</summary>
|
||||
public int KillCount { get; private set; }
|
||||
|
||||
/// <summary>Gets the number of times dispose was called on the fake worker client.</summary>
|
||||
public int DisposeCount { get; private set; }
|
||||
|
||||
/// <summary>Gets the exception to throw when shutdown is called, if any.</summary>
|
||||
public Exception? ShutdownException { get; init; }
|
||||
|
||||
/// <summary>Gets a value indicating whether to block shutdown on the fake worker client.</summary>
|
||||
public bool BlockShutdown { get; init; }
|
||||
|
||||
/// <summary>Gets the last command invoked on the fake worker client.</summary>
|
||||
public WorkerCommand? LastCommand { get; private set; }
|
||||
|
||||
/// <summary>Gets the reply to return for invoke calls on the fake worker client.</summary>
|
||||
public WorkerCommandReply? InvokeReply { get; init; }
|
||||
|
||||
private TaskCompletionSource ShutdownStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
private TaskCompletionSource ShutdownReleased { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerCommandReply> InvokeAsync(
|
||||
WorkerCommand command,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
InvokeCount++;
|
||||
LastCommand = command;
|
||||
if (InvokeReply is not null)
|
||||
{
|
||||
return Task.FromResult(InvokeReply);
|
||||
}
|
||||
|
||||
MxCommandKind kind = command.Command?.Kind ?? MxCommandKind.Unspecified;
|
||||
|
||||
return Task.FromResult(new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
CorrelationId = "correlation-1",
|
||||
Kind = kind,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task ShutdownAsync(
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ShutdownCount++;
|
||||
if (ShutdownException is not null)
|
||||
{
|
||||
throw ShutdownException;
|
||||
}
|
||||
|
||||
if (BlockShutdown)
|
||||
{
|
||||
ShutdownStarted.TrySetResult();
|
||||
await ShutdownReleased.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
State = WorkerClientState.Closed;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Kill(string reason)
|
||||
{
|
||||
KillCount++;
|
||||
State = WorkerClientState.Faulted;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
DisposeCount++;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Waits for shutdown to start on the fake worker client.</summary>
|
||||
public Task WaitForShutdownStartAsync()
|
||||
{
|
||||
return ShutdownStarted.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
/// <summary>Releases the shutdown block on the fake worker client.</summary>
|
||||
public void ReleaseShutdown()
|
||||
{
|
||||
ShutdownReleased.TrySetResult();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+391
@@ -0,0 +1,391 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
|
||||
|
||||
public sealed class SessionWorkerClientFactoryFakeWorkerTests : IAsyncDisposable
|
||||
{
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
private readonly List<IWorkerTaskLauncher> _launchers = [];
|
||||
|
||||
/// <summary>
|
||||
/// Awaits every scripted worker task so an unhandled exception fails the owning test
|
||||
/// instead of surfacing later as an unobserved <see cref="TaskScheduler.UnobservedTaskException"/>.
|
||||
/// </summary>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
foreach (IWorkerTaskLauncher launcher in _launchers)
|
||||
{
|
||||
await launcher.ObserveWorkerTaskAsync(TestTimeout);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the factory creates a ready worker client with a scripted fake worker process.</summary>
|
||||
[Fact]
|
||||
public async Task CreateAsync_WithScriptedFakeWorker_ReturnsReadyClient()
|
||||
{
|
||||
ScriptedFakeWorkerProcessLauncher launcher = Track(new ScriptedFakeWorkerProcessLauncher());
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionWorkerClientFactory factory = new(
|
||||
launcher,
|
||||
Options.Create(CreateOptions()),
|
||||
metrics,
|
||||
NullLoggerFactory.Instance);
|
||||
GatewaySession session = CreateSession();
|
||||
|
||||
await using IWorkerClient workerClient = await factory.CreateAsync(
|
||||
session,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(WorkerClientState.Ready, workerClient.State);
|
||||
Assert.Equal(ScriptedFakeWorkerProcessLauncher.ProcessId, workerClient.ProcessId);
|
||||
Assert.NotNull(launcher.Harness);
|
||||
|
||||
Task<WorkerCommandReply> invokeTask = workerClient.InvokeAsync(
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
TestTimeout,
|
||||
CancellationToken.None);
|
||||
WorkerEnvelope commandEnvelope = await launcher.Harness.ReadCommandAsync();
|
||||
await launcher.Harness.ReplyToCommandAsync(commandEnvelope);
|
||||
WorkerCommandReply reply = await invokeTask.WaitAsync(TestTimeout);
|
||||
|
||||
Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, reply.Reply.ProtocolStatus.Code);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a failed fake worker startup throws a worker client exception.</summary>
|
||||
[Fact]
|
||||
public async Task CreateAsync_WhenFakeWorkerStartupFails_ThrowsWorkerClientException()
|
||||
{
|
||||
FailingStartupWorkerProcessLauncher launcher = Track(new FailingStartupWorkerProcessLauncher());
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionWorkerClientFactory factory = new(
|
||||
launcher,
|
||||
Options.Create(CreateOptions()),
|
||||
metrics,
|
||||
NullLoggerFactory.Instance);
|
||||
GatewaySession session = CreateSession();
|
||||
|
||||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||||
async () => await factory.CreateAsync(session, CancellationToken.None).WaitAsync(TestTimeout));
|
||||
|
||||
Assert.Equal(WorkerClientErrorCode.ProtocolViolation, exception.ErrorCode);
|
||||
Assert.True(launcher.Process.IsDisposed);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a worker that never sends ready times out and is killed.</summary>
|
||||
[Fact]
|
||||
public async Task CreateAsync_WhenFakeWorkerNeverSendsReady_TimesOutAndKillsWorker()
|
||||
{
|
||||
NeverReadyWorkerProcessLauncher launcher = Track(new NeverReadyWorkerProcessLauncher());
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionWorkerClientFactory factory = new(
|
||||
launcher,
|
||||
Options.Create(CreateOptions(startupTimeoutSeconds: 1)),
|
||||
metrics,
|
||||
NullLoggerFactory.Instance);
|
||||
GatewaySession session = CreateSession(startupTimeout: TimeSpan.FromSeconds(1));
|
||||
|
||||
TimeoutException exception = await Assert.ThrowsAsync<TimeoutException>(
|
||||
async () => await factory.CreateAsync(session, CancellationToken.None).WaitAsync(TestTimeout));
|
||||
|
||||
Assert.Contains("did not complete startup", exception.Message);
|
||||
Assert.Equal(1, launcher.Process.KillCount);
|
||||
Assert.True(launcher.Process.IsDisposed);
|
||||
}
|
||||
|
||||
private static GatewayOptions CreateOptions(int startupTimeoutSeconds = 5)
|
||||
{
|
||||
return new GatewayOptions
|
||||
{
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
StartupTimeoutSeconds = startupTimeoutSeconds,
|
||||
ShutdownTimeoutSeconds = 5,
|
||||
HeartbeatIntervalSeconds = 30,
|
||||
HeartbeatGraceSeconds = 30,
|
||||
MaxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes,
|
||||
},
|
||||
Events = new EventOptions
|
||||
{
|
||||
QueueCapacity = 16,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static GatewaySession CreateSession(TimeSpan? startupTimeout = null)
|
||||
{
|
||||
return new GatewaySession(
|
||||
FakeWorkerHarness.DefaultSessionId,
|
||||
GatewayContractInfo.DefaultBackendName,
|
||||
$"mxaccessgw-session-fake-worker-{Guid.NewGuid():N}",
|
||||
FakeWorkerHarness.DefaultNonce,
|
||||
"test-client",
|
||||
"fake-worker-session-test",
|
||||
"client-correlation-1",
|
||||
startupTimeout ?? TestTimeout,
|
||||
TestTimeout,
|
||||
TestTimeout,
|
||||
DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static WorkerCommand CreateCommand(MxCommandKind kind)
|
||||
{
|
||||
return new WorkerCommand
|
||||
{
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = kind,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private T Track<T>(T launcher)
|
||||
where T : IWorkerTaskLauncher
|
||||
{
|
||||
_launchers.Add(launcher);
|
||||
|
||||
return launcher;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A fake worker launcher that runs a scripted worker on a background task and exposes
|
||||
/// that task so the owning test observes it rather than leaking an unobserved fault.
|
||||
/// </summary>
|
||||
private interface IWorkerTaskLauncher : IWorkerProcessLauncher
|
||||
{
|
||||
/// <summary>
|
||||
/// Awaits the scripted worker task within the timeout, swallowing only the pipe
|
||||
/// teardown faults expected when the worker client kills or disposes the worker.
|
||||
/// </summary>
|
||||
/// <param name="timeout">Maximum time to wait for the worker task.</param>
|
||||
Task ObserveWorkerTaskAsync(TimeSpan timeout);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Awaits a scripted worker task, treating cancellation and pipe-disconnect I/O faults as
|
||||
/// the expected outcome of the worker client tearing the worker down, and rethrowing anything else.
|
||||
/// </summary>
|
||||
private static async Task ObserveWorkerTaskAsync(Task workerTask, TimeSpan timeout)
|
||||
{
|
||||
try
|
||||
{
|
||||
await workerTask.WaitAsync(timeout).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Expected: the worker client cancelled the scripted worker during teardown.
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Expected: the gateway pipe was closed when the worker client disposed.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake worker launcher that connects a scripted fake worker harness.</summary>
|
||||
private sealed class ScriptedFakeWorkerProcessLauncher : IWorkerTaskLauncher
|
||||
{
|
||||
/// <summary>The fake process ID used by the scripted launcher.</summary>
|
||||
public const int ProcessId = 2468;
|
||||
private readonly FakeWorkerProcess _process = new(ProcessId);
|
||||
|
||||
/// <summary>Gets the connected fake worker harness.</summary>
|
||||
public FakeWorkerHarness? Harness { get; private set; }
|
||||
|
||||
/// <summary>Gets the scripted worker task.</summary>
|
||||
public Task WorkerTask { get; private set; } = Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerProcessHandle> LaunchAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
WorkerTask = RunWorkerAsync(request, cancellationToken);
|
||||
|
||||
return Task.FromResult(CreateHandle(_process));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ObserveWorkerTaskAsync(TimeSpan timeout) =>
|
||||
SessionWorkerClientFactoryFakeWorkerTests.ObserveWorkerTaskAsync(WorkerTask, timeout);
|
||||
|
||||
private async Task RunWorkerAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Harness = await FakeWorkerHarness.ConnectToGatewayPipeAsync(
|
||||
request.SessionId,
|
||||
request.Nonce,
|
||||
request.PipeName,
|
||||
request.ProtocolVersion,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
await Harness.CompleteStartupAsync(ProcessId, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake worker launcher that fails during startup with protocol version mismatch.</summary>
|
||||
private sealed class FailingStartupWorkerProcessLauncher : IWorkerTaskLauncher
|
||||
{
|
||||
/// <summary>Gets the fake worker process.</summary>
|
||||
public FakeWorkerProcess Process { get; } = new(processId: 3579);
|
||||
|
||||
/// <summary>Gets the scripted worker task.</summary>
|
||||
public Task WorkerTask { get; private set; } = Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerProcessHandle> LaunchAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
WorkerTask = RunWorkerAsync(request, cancellationToken);
|
||||
|
||||
return Task.FromResult(CreateHandle(Process));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ObserveWorkerTaskAsync(TimeSpan timeout) =>
|
||||
SessionWorkerClientFactoryFakeWorkerTests.ObserveWorkerTaskAsync(WorkerTask, timeout);
|
||||
|
||||
private async Task RunWorkerAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using FakeWorkerHarness harness = await FakeWorkerHarness.ConnectToGatewayPipeAsync(
|
||||
request.SessionId,
|
||||
request.Nonce,
|
||||
request.PipeName,
|
||||
request.ProtocolVersion,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
_ = await harness.ReadGatewayEnvelopeAsync(cancellationToken).ConfigureAwait(false);
|
||||
await harness.SendWorkerHelloAsync(
|
||||
workerProcessId: Process.Id,
|
||||
workerProtocolVersion: request.ProtocolVersion + 1,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake worker launcher that never completes startup, simulating a hung worker.</summary>
|
||||
private sealed class NeverReadyWorkerProcessLauncher : IWorkerTaskLauncher
|
||||
{
|
||||
private readonly CancellationTokenSource _stop = new();
|
||||
|
||||
/// <summary>Gets the fake worker process.</summary>
|
||||
public FakeWorkerProcess Process { get; } = new(processId: 4680);
|
||||
|
||||
/// <summary>Gets the scripted worker task.</summary>
|
||||
public Task WorkerTask { get; private set; } = Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerProcessHandle> LaunchAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
WorkerTask = RunWorkerAsync(request);
|
||||
|
||||
return Task.FromResult(CreateHandle(Process));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task ObserveWorkerTaskAsync(TimeSpan timeout)
|
||||
{
|
||||
// The scripted worker parks on an infinite delay; cancel it so disposal observes
|
||||
// the task instead of leaking it as an unobserved fault.
|
||||
await _stop.CancelAsync().ConfigureAwait(false);
|
||||
await SessionWorkerClientFactoryFakeWorkerTests
|
||||
.ObserveWorkerTaskAsync(WorkerTask, timeout)
|
||||
.ConfigureAwait(false);
|
||||
_stop.Dispose();
|
||||
}
|
||||
|
||||
private async Task RunWorkerAsync(WorkerProcessLaunchRequest request)
|
||||
{
|
||||
await using FakeWorkerHarness harness = await FakeWorkerHarness.ConnectToGatewayPipeAsync(
|
||||
request.SessionId,
|
||||
request.Nonce,
|
||||
request.PipeName,
|
||||
request.ProtocolVersion,
|
||||
cancellationToken: _stop.Token).ConfigureAwait(false);
|
||||
_ = await harness.ReadGatewayEnvelopeAsync(_stop.Token).ConfigureAwait(false);
|
||||
await harness.SendWorkerHelloAsync(
|
||||
workerProcessId: Process.Id,
|
||||
workerProtocolVersion: request.ProtocolVersion,
|
||||
cancellationToken: _stop.Token).ConfigureAwait(false);
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, _stop.Token).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static WorkerProcessHandle CreateHandle(IWorkerProcess process)
|
||||
{
|
||||
return new WorkerProcessHandle(
|
||||
process,
|
||||
new WorkerProcessCommandLine("fake-worker.exe", []),
|
||||
DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fake worker process for testing process lifecycle. <see cref="WaitForExitAsync"/>
|
||||
/// awaits a <see cref="TaskCompletionSource"/> completed only by
|
||||
/// <see cref="Kill"/> or <see cref="MarkExited"/>, so a caller observing
|
||||
/// completion can trust that exit actually happened — bringing this fake into
|
||||
/// parity with the smoke-test variant in <c>GatewayEndToEndFakeWorkerSmokeTests</c>
|
||||
/// (Tests-015 / Tests-023). This removes the latent regression vector where a
|
||||
/// future <c>Assert.True(launcher.Process.HasExited)</c> in this file would
|
||||
/// pass spuriously regardless of whether the worker truly exited.
|
||||
/// </summary>
|
||||
private sealed class FakeWorkerProcess(int processId) : IWorkerProcess
|
||||
{
|
||||
private readonly TaskCompletionSource _exited = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private bool _disposed;
|
||||
|
||||
/// <inheritdoc />
|
||||
public int Id { get; } = processId;
|
||||
|
||||
/// <summary>Gets a value indicating whether the process has exited.</summary>
|
||||
public bool HasExited { get; private set; }
|
||||
|
||||
/// <summary>Gets the process exit code, or null if the process has not exited.</summary>
|
||||
public int? ExitCode { get; private set; }
|
||||
|
||||
/// <summary>Gets the number of times the Kill method was called.</summary>
|
||||
public int KillCount { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask WaitForExitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return new ValueTask(_exited.Task.WaitAsync(cancellationToken));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Kill(bool entireProcessTree)
|
||||
{
|
||||
KillCount++;
|
||||
MarkExited(-1);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
_disposed = true;
|
||||
}
|
||||
|
||||
/// <summary>Gets a value indicating whether this process has been disposed.</summary>
|
||||
public bool IsDisposed => _disposed;
|
||||
|
||||
/// <summary>Marks the process as exited with the specified exit code.</summary>
|
||||
/// <param name="exitCode">The process exit code.</param>
|
||||
public void MarkExited(int exitCode)
|
||||
{
|
||||
HasExited = true;
|
||||
ExitCode = exitCode;
|
||||
_exited.TrySetResult();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
|
||||
|
||||
public sealed class FakeWorkerHarnessTests
|
||||
{
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Verifies that completing startup with hello and ready transitions the client to ready state.</summary>
|
||||
[Fact]
|
||||
public async Task CompleteStartupAsync_WithHelloAndReady_TransitionsClientToReady()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = fakeWorker.CreateClient();
|
||||
|
||||
Task startTask = client.StartAsync(CancellationToken.None);
|
||||
WorkerEnvelope gatewayHello = await fakeWorker.CompleteStartupAsync();
|
||||
await startTask.WaitAsync(TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, gatewayHello.BodyCase);
|
||||
Assert.Equal(FakeWorkerHarness.DefaultNonce, gatewayHello.GatewayHello.Nonce);
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
Assert.Equal(FakeWorkerHarness.DefaultWorkerProcessId, client.ProcessId);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a protocol version mismatch during startup fails the client.</summary>
|
||||
[Fact]
|
||||
public async Task StartAsync_WithProtocolMismatch_FailsStartup()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = fakeWorker.CreateClient();
|
||||
|
||||
Task startTask = client.StartAsync(CancellationToken.None);
|
||||
WorkerEnvelope gatewayHello = await fakeWorker.ReadGatewayEnvelopeAsync();
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, gatewayHello.BodyCase);
|
||||
await fakeWorker.SendWorkerHelloAsync(
|
||||
workerProtocolVersion: GatewayContractInfo.WorkerProtocolVersion + 1);
|
||||
|
||||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||||
async () => await startTask.WaitAsync(TestTimeout));
|
||||
|
||||
Assert.Equal(WorkerClientErrorCode.ProtocolViolation, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a scripted reply completes a pending command invocation.</summary>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WithScriptedReply_CompletesCommand()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = fakeWorker.CreateClient();
|
||||
await StartClientAsync(fakeWorker, client);
|
||||
|
||||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
TestTimeout,
|
||||
CancellationToken.None);
|
||||
WorkerEnvelope commandEnvelope = await fakeWorker.ReadCommandAsync();
|
||||
await fakeWorker.ReplyToCommandAsync(commandEnvelope);
|
||||
|
||||
WorkerCommandReply reply = await invokeTask.WaitAsync(TestTimeout);
|
||||
|
||||
Assert.Equal(commandEnvelope.CorrelationId, reply.Reply.CorrelationId);
|
||||
Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, reply.Reply.ProtocolStatus.Code);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that scripted events are yielded in order through the event stream.</summary>
|
||||
[Fact]
|
||||
public async Task ReadEventsAsync_WithScriptedEvents_YieldsOrderedEvents()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = fakeWorker.CreateClient();
|
||||
await StartClientAsync(fakeWorker, client);
|
||||
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
|
||||
|
||||
await using IAsyncEnumerator<WorkerEvent> events =
|
||||
client.ReadEventsAsync(cancellationTokenSource.Token).GetAsyncEnumerator(cancellationTokenSource.Token);
|
||||
|
||||
await fakeWorker.EmitEventAsync(MxEventFamily.OnDataChange, cancellationTokenSource.Token);
|
||||
await fakeWorker.EmitEventAsync(MxEventFamily.OperationComplete, cancellationTokenSource.Token);
|
||||
|
||||
Assert.True(await events.MoveNextAsync());
|
||||
Assert.Equal((ulong)3, events.Current.Event.WorkerSequence);
|
||||
Assert.Equal(MxEventFamily.OnDataChange, events.Current.Event.Family);
|
||||
|
||||
Assert.True(await events.MoveNextAsync());
|
||||
Assert.Equal((ulong)4, events.Current.Event.WorkerSequence);
|
||||
Assert.Equal(MxEventFamily.OperationComplete, events.Current.Event.Family);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a scripted fault from the worker faults the client.</summary>
|
||||
[Fact]
|
||||
public async Task ReadLoop_WithScriptedFault_FaultsClient()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = fakeWorker.CreateClient();
|
||||
await StartClientAsync(fakeWorker, client);
|
||||
|
||||
await fakeWorker.EmitFaultAsync(
|
||||
WorkerFaultCategory.MxaccessCommandFailed,
|
||||
"scripted MXAccess command fault");
|
||||
|
||||
await WaitUntilAsync(
|
||||
() => client.State == WorkerClientState.Faulted,
|
||||
TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that sending a heartbeat updates the client heartbeat state. Uses a
|
||||
/// <see cref="ManualTimeProvider"/> so the timestamp advance is deterministic rather
|
||||
/// than relying on a wall-clock <c>Task.Delay</c> exceeding clock resolution.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task SendHeartbeatAsync_UpdatesClientHeartbeatState()
|
||||
{
|
||||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-18T12:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = fakeWorker.CreateClient(timeProvider: clock);
|
||||
await StartClientAsync(fakeWorker, client);
|
||||
DateTimeOffset previousHeartbeat = client.LastHeartbeatAt;
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
await fakeWorker.SendHeartbeatAsync(
|
||||
configureHeartbeat: heartbeat => heartbeat.WorkerProcessId = 2468);
|
||||
|
||||
await WaitUntilAsync(
|
||||
() => client.ProcessId == 2468 && client.LastHeartbeatAt > previousHeartbeat,
|
||||
TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
Assert.Equal(previousHeartbeat + TimeSpan.FromSeconds(1), client.LastHeartbeatAt);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a hung worker times out pending command invocations.</summary>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WithHungWorker_TimesOutPendingCommand()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = fakeWorker.CreateClient();
|
||||
await StartClientAsync(fakeWorker, client);
|
||||
|
||||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
TimeSpan.FromMilliseconds(50),
|
||||
CancellationToken.None);
|
||||
WorkerEnvelope commandEnvelope = await fakeWorker.ReadCommandAsync();
|
||||
|
||||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||||
async () => await invokeTask.WaitAsync(TestTimeout));
|
||||
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||||
Assert.Equal(WorkerClientErrorCode.CommandTimeout, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a malformed frame in the read loop faults the client.</summary>
|
||||
[Fact]
|
||||
public async Task ReadLoop_WithMalformedFrame_FaultsClient()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = fakeWorker.CreateClient();
|
||||
await StartClientAsync(fakeWorker, client);
|
||||
|
||||
await fakeWorker.WriteMalformedPayloadAsync(new byte[] { 0x08, 0x96, 0x01 });
|
||||
|
||||
await WaitUntilAsync(
|
||||
() => client.State == WorkerClientState.Faulted,
|
||||
TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a shutdown acknowledgment from the worker closes the client.</summary>
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_WithShutdownAck_ClosesClient()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = fakeWorker.CreateClient();
|
||||
await StartClientAsync(fakeWorker, client);
|
||||
|
||||
Task shutdownTask = client.ShutdownAsync(TestTimeout, CancellationToken.None);
|
||||
WorkerEnvelope shutdownEnvelope = await fakeWorker.ReadShutdownAsync();
|
||||
await fakeWorker.SendShutdownAckAsync();
|
||||
await shutdownTask.WaitAsync(TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerShutdown, shutdownEnvelope.BodyCase);
|
||||
Assert.Equal(WorkerClientState.Closed, client.State);
|
||||
}
|
||||
|
||||
private static async Task StartClientAsync(
|
||||
FakeWorkerHarness fakeWorker,
|
||||
WorkerClient client)
|
||||
{
|
||||
Task startTask = client.StartAsync(CancellationToken.None);
|
||||
await fakeWorker.CompleteStartupAsync().ConfigureAwait(false);
|
||||
await startTask.WaitAsync(TestTimeout).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static WorkerCommand CreateCommand(MxCommandKind kind)
|
||||
{
|
||||
return new WorkerCommand
|
||||
{
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = kind,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(
|
||||
Func<bool> predicate,
|
||||
TimeSpan timeout)
|
||||
{
|
||||
using CancellationTokenSource cancellationTokenSource = new(timeout);
|
||||
while (!predicate())
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,483 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.IO.Pipes;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
|
||||
|
||||
public sealed class FakeWorkerHarness : IAsyncDisposable
|
||||
{
|
||||
public const string DefaultSessionId = "session-fake-worker";
|
||||
public const string DefaultNonce = "nonce-fake-worker";
|
||||
public const int DefaultWorkerProcessId = 9321;
|
||||
|
||||
private readonly NamedPipeServerStream? _gatewayStream;
|
||||
private readonly NamedPipeClientStream _workerStream;
|
||||
private readonly WorkerFrameProtocolOptions _frameOptions;
|
||||
private readonly WorkerFrameReader _reader;
|
||||
private readonly WorkerFrameWriter _writer;
|
||||
private bool _workerSideDisposed;
|
||||
|
||||
private FakeWorkerHarness(
|
||||
string sessionId,
|
||||
string nonce,
|
||||
NamedPipeServerStream? gatewayStream,
|
||||
NamedPipeClientStream workerStream,
|
||||
WorkerFrameProtocolOptions frameOptions)
|
||||
{
|
||||
SessionId = sessionId;
|
||||
Nonce = nonce;
|
||||
_gatewayStream = gatewayStream;
|
||||
_workerStream = workerStream;
|
||||
_frameOptions = frameOptions;
|
||||
_reader = new WorkerFrameReader(_workerStream, frameOptions);
|
||||
_writer = new WorkerFrameWriter(_workerStream, frameOptions);
|
||||
}
|
||||
|
||||
/// <summary>Gets the session ID for the fake worker harness.</summary>
|
||||
public string SessionId { get; }
|
||||
|
||||
/// <summary>Gets the nonce for the fake worker harness.</summary>
|
||||
public string Nonce { get; }
|
||||
|
||||
/// <summary>Gets or sets the next worker sequence number.</summary>
|
||||
public ulong NextWorkerSequence { get; private set; }
|
||||
|
||||
/// <summary>Creates a connected pair of fake worker harness with gateway and worker pipes.</summary>
|
||||
/// <param name="sessionId">Identifier for the fake session.</param>
|
||||
/// <param name="nonce">Nonce for session validation.</param>
|
||||
/// <param name="protocolVersion">Protocol version for frame communication.</param>
|
||||
/// <param name="maxMessageBytes">Maximum message size in bytes.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
public static async Task<FakeWorkerHarness> CreateConnectedPairAsync(
|
||||
string sessionId = DefaultSessionId,
|
||||
string nonce = DefaultNonce,
|
||||
uint protocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||||
int maxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string pipeName = $"mxaccessgw-fake-worker-{Guid.NewGuid():N}";
|
||||
NamedPipeServerStream gatewayStream = new(
|
||||
pipeName,
|
||||
PipeDirection.InOut,
|
||||
maxNumberOfServerInstances: 1,
|
||||
PipeTransmissionMode.Byte,
|
||||
PipeOptions.Asynchronous);
|
||||
NamedPipeClientStream workerStream = CreateWorkerStream(pipeName);
|
||||
|
||||
Task waitForConnectionTask = gatewayStream.WaitForConnectionAsync(cancellationToken);
|
||||
await workerStream.ConnectAsync(cancellationToken).ConfigureAwait(false);
|
||||
await waitForConnectionTask.ConfigureAwait(false);
|
||||
|
||||
return new FakeWorkerHarness(
|
||||
sessionId,
|
||||
nonce,
|
||||
gatewayStream,
|
||||
workerStream,
|
||||
new WorkerFrameProtocolOptions(sessionId, protocolVersion, maxMessageBytes));
|
||||
}
|
||||
|
||||
/// <summary>Connects to an existing gateway pipe as a fake worker harness.</summary>
|
||||
/// <param name="sessionId">Identifier for the fake session.</param>
|
||||
/// <param name="nonce">Nonce for session validation.</param>
|
||||
/// <param name="pipeName">Name of the named pipe to connect to.</param>
|
||||
/// <param name="protocolVersion">Protocol version for frame communication.</param>
|
||||
/// <param name="maxMessageBytes">Maximum message size in bytes.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
public static async Task<FakeWorkerHarness> ConnectToGatewayPipeAsync(
|
||||
string sessionId,
|
||||
string nonce,
|
||||
string pipeName,
|
||||
uint protocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||||
int maxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
NamedPipeClientStream workerStream = CreateWorkerStream(pipeName);
|
||||
await workerStream.ConnectAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return new FakeWorkerHarness(
|
||||
sessionId,
|
||||
nonce,
|
||||
gatewayStream: null,
|
||||
workerStream,
|
||||
new WorkerFrameProtocolOptions(sessionId, protocolVersion, maxMessageBytes));
|
||||
}
|
||||
|
||||
/// <summary>Creates a worker client connected to the fake worker harness.</summary>
|
||||
/// <param name="options">Configuration options for the worker client.</param>
|
||||
/// <param name="metrics">Gateway metrics collector.</param>
|
||||
/// <param name="timeProvider">Time provider for timestamps.</param>
|
||||
/// <returns>A configured worker client connected to this harness.</returns>
|
||||
public WorkerClient CreateClient(
|
||||
WorkerClientOptions? options = null,
|
||||
GatewayMetrics? metrics = null,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
if (_gatewayStream is null)
|
||||
{
|
||||
throw new InvalidOperationException("This fake worker is connected to a gateway-owned pipe.");
|
||||
}
|
||||
|
||||
WorkerClientConnection connection = new(
|
||||
SessionId,
|
||||
Nonce,
|
||||
_gatewayStream,
|
||||
_frameOptions);
|
||||
|
||||
return new WorkerClient(connection, options, metrics, timeProvider);
|
||||
}
|
||||
|
||||
/// <summary>Completes the worker startup handshake by reading the gateway hello and sending worker hello and ready.</summary>
|
||||
/// <param name="workerProcessId">Process ID of the fake worker.</param>
|
||||
/// <param name="workerVersion">Version string of the fake worker.</param>
|
||||
/// <param name="mxaccessProgid">MXAccess COM ProgID.</param>
|
||||
/// <param name="mxaccessClsid">MXAccess COM CLSID.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>The gateway hello envelope received during startup.</returns>
|
||||
public async Task<WorkerEnvelope> CompleteStartupAsync(
|
||||
int workerProcessId = DefaultWorkerProcessId,
|
||||
string workerVersion = "fake-worker",
|
||||
string mxaccessProgid = "LMXProxy.LMXProxyServer.1",
|
||||
string mxaccessClsid = "{C30B52F5-2CB5-4760-AF0A-3A344A7EB5DC}",
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
WorkerEnvelope gatewayHello = await ReadGatewayEnvelopeAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (gatewayHello.BodyCase != WorkerEnvelope.BodyOneofCase.GatewayHello)
|
||||
{
|
||||
throw new InvalidOperationException($"Expected GatewayHello but received {gatewayHello.BodyCase}.");
|
||||
}
|
||||
|
||||
await SendWorkerHelloAsync(
|
||||
workerProcessId,
|
||||
workerVersion,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
await SendWorkerReadyAsync(
|
||||
workerProcessId,
|
||||
mxaccessProgid,
|
||||
mxaccessClsid,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return gatewayHello;
|
||||
}
|
||||
|
||||
/// <summary>Reads the next gateway envelope from the worker stream.</summary>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>The gateway envelope read from the stream.</returns>
|
||||
public async Task<WorkerEnvelope> ReadGatewayEnvelopeAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await _reader.ReadAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Reads the next command from the worker stream.</summary>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>The command envelope read from the stream.</returns>
|
||||
public async Task<WorkerEnvelope> ReadCommandAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
WorkerEnvelope envelope = await ReadGatewayEnvelopeAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (envelope.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCommand)
|
||||
{
|
||||
throw new InvalidOperationException($"Expected WorkerCommand but received {envelope.BodyCase}.");
|
||||
}
|
||||
|
||||
return envelope;
|
||||
}
|
||||
|
||||
/// <summary>Reads the next shutdown request from the worker stream.</summary>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>The shutdown envelope read from the stream.</returns>
|
||||
public async Task<WorkerEnvelope> ReadShutdownAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
WorkerEnvelope envelope = await ReadGatewayEnvelopeAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (envelope.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerShutdown)
|
||||
{
|
||||
throw new InvalidOperationException($"Expected WorkerShutdown but received {envelope.BodyCase}.");
|
||||
}
|
||||
|
||||
return envelope;
|
||||
}
|
||||
|
||||
/// <summary>Sends a worker hello message to the gateway.</summary>
|
||||
/// <param name="workerProcessId">Process ID of the fake worker.</param>
|
||||
/// <param name="workerVersion">Version string of the fake worker.</param>
|
||||
/// <param name="workerProtocolVersion">Protocol version override.</param>
|
||||
/// <param name="nonce">Nonce override.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
public async Task SendWorkerHelloAsync(
|
||||
int workerProcessId = DefaultWorkerProcessId,
|
||||
string workerVersion = "fake-worker",
|
||||
uint? workerProtocolVersion = null,
|
||||
string? nonce = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _writer.WriteAsync(
|
||||
CreateEnvelope(
|
||||
correlationId: string.Empty,
|
||||
envelope => envelope.WorkerHello = new WorkerHello
|
||||
{
|
||||
ProtocolVersion = workerProtocolVersion ?? _frameOptions.ProtocolVersion,
|
||||
Nonce = nonce ?? Nonce,
|
||||
WorkerProcessId = workerProcessId,
|
||||
WorkerVersion = workerVersion,
|
||||
}),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Sends a worker ready message to the gateway.</summary>
|
||||
/// <param name="workerProcessId">Process ID of the fake worker.</param>
|
||||
/// <param name="mxaccessProgid">MXAccess COM ProgID.</param>
|
||||
/// <param name="mxaccessClsid">MXAccess COM CLSID.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
public async Task SendWorkerReadyAsync(
|
||||
int workerProcessId = DefaultWorkerProcessId,
|
||||
string mxaccessProgid = "LMXProxy.LMXProxyServer.1",
|
||||
string mxaccessClsid = "{C30B52F5-2CB5-4760-AF0A-3A344A7EB5DC}",
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _writer.WriteAsync(
|
||||
CreateEnvelope(
|
||||
correlationId: string.Empty,
|
||||
envelope => envelope.WorkerReady = new WorkerReady
|
||||
{
|
||||
WorkerProcessId = workerProcessId,
|
||||
MxaccessProgid = mxaccessProgid,
|
||||
MxaccessClsid = mxaccessClsid,
|
||||
ReadyTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
|
||||
}),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Sends a reply to a command received from the gateway.</summary>
|
||||
/// <param name="commandEnvelope">The command envelope to reply to.</param>
|
||||
/// <param name="statusCode">Protocol status code for the reply.</param>
|
||||
/// <param name="statusMessage">Human-readable status message.</param>
|
||||
/// <param name="configureReply">Optional callback to customize the reply.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
public async Task ReplyToCommandAsync(
|
||||
WorkerEnvelope commandEnvelope,
|
||||
ProtocolStatusCode statusCode = ProtocolStatusCode.Ok,
|
||||
string statusMessage = "OK",
|
||||
Action<MxCommandReply>? configureReply = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (commandEnvelope.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCommand)
|
||||
{
|
||||
throw new ArgumentException("Command envelope must contain WorkerCommand.", nameof(commandEnvelope));
|
||||
}
|
||||
|
||||
MxCommandKind kind = commandEnvelope.WorkerCommand.Command?.Kind ?? MxCommandKind.Unspecified;
|
||||
MxCommandReply reply = new()
|
||||
{
|
||||
SessionId = SessionId,
|
||||
CorrelationId = commandEnvelope.CorrelationId,
|
||||
Kind = kind,
|
||||
ProtocolStatus = new ProtocolStatus
|
||||
{
|
||||
Code = statusCode,
|
||||
Message = statusMessage,
|
||||
},
|
||||
};
|
||||
configureReply?.Invoke(reply);
|
||||
|
||||
await _writer.WriteAsync(
|
||||
CreateEnvelope(
|
||||
commandEnvelope.CorrelationId,
|
||||
envelope => envelope.WorkerCommandReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = reply,
|
||||
CompletedTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
|
||||
}),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Emits an event to the gateway.</summary>
|
||||
/// <param name="family">Family of the event to emit.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <param name="configureEvent">Optional callback to customize the event.</param>
|
||||
public async Task EmitEventAsync(
|
||||
MxEventFamily family,
|
||||
CancellationToken cancellationToken = default,
|
||||
Action<MxEvent>? configureEvent = null)
|
||||
{
|
||||
ulong sequence = NextWorkerSequence + 1;
|
||||
MxEvent mxEvent = new()
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Family = family,
|
||||
WorkerSequence = sequence,
|
||||
WorkerTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
|
||||
};
|
||||
configureEvent?.Invoke(mxEvent);
|
||||
|
||||
await _writer.WriteAsync(
|
||||
CreateEnvelope(
|
||||
correlationId: string.Empty,
|
||||
envelope => envelope.WorkerEvent = new WorkerEvent
|
||||
{
|
||||
Event = mxEvent,
|
||||
}),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Emits a fault message to the gateway.</summary>
|
||||
/// <param name="category">Category of the fault.</param>
|
||||
/// <param name="diagnosticMessage">Diagnostic message describing the fault.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
public async Task EmitFaultAsync(
|
||||
WorkerFaultCategory category,
|
||||
string diagnosticMessage,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _writer.WriteAsync(
|
||||
CreateEnvelope(
|
||||
correlationId: string.Empty,
|
||||
envelope => envelope.WorkerFault = new WorkerFault
|
||||
{
|
||||
Category = category,
|
||||
DiagnosticMessage = diagnosticMessage,
|
||||
ProtocolStatus = new ProtocolStatus
|
||||
{
|
||||
Code = ProtocolStatusCode.WorkerUnavailable,
|
||||
Message = diagnosticMessage,
|
||||
},
|
||||
}),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Sends a heartbeat message to the gateway.</summary>
|
||||
/// <param name="state">Current worker state.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <param name="configureHeartbeat">Optional callback to customize the heartbeat.</param>
|
||||
public async Task SendHeartbeatAsync(
|
||||
WorkerState state = WorkerState.Ready,
|
||||
CancellationToken cancellationToken = default,
|
||||
Action<WorkerHeartbeat>? configureHeartbeat = null)
|
||||
{
|
||||
WorkerHeartbeat heartbeat = new()
|
||||
{
|
||||
WorkerProcessId = DefaultWorkerProcessId,
|
||||
State = state,
|
||||
LastStaActivityTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
|
||||
};
|
||||
configureHeartbeat?.Invoke(heartbeat);
|
||||
|
||||
await _writer.WriteAsync(
|
||||
CreateEnvelope(
|
||||
correlationId: string.Empty,
|
||||
envelope => envelope.WorkerHeartbeat = heartbeat),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Sends a shutdown acknowledgment message to the gateway.</summary>
|
||||
/// <param name="statusCode">Protocol status code for the acknowledgment.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
public async Task SendShutdownAckAsync(
|
||||
ProtocolStatusCode statusCode = ProtocolStatusCode.Ok,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _writer.WriteAsync(
|
||||
CreateEnvelope(
|
||||
correlationId: string.Empty,
|
||||
envelope => envelope.WorkerShutdownAck = new WorkerShutdownAck
|
||||
{
|
||||
Status = new ProtocolStatus
|
||||
{
|
||||
Code = statusCode,
|
||||
Message = statusCode.ToString(),
|
||||
},
|
||||
}),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Writes a malformed payload directly to the worker stream.</summary>
|
||||
/// <param name="payload">Malformed payload bytes to write.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
public async Task WriteMalformedPayloadAsync(
|
||||
ReadOnlyMemory<byte> payload,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (payload.IsEmpty)
|
||||
{
|
||||
throw new ArgumentException("Malformed payload must include at least one byte.", nameof(payload));
|
||||
}
|
||||
|
||||
byte[] lengthPrefix = new byte[sizeof(uint)];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(lengthPrefix, (uint)payload.Length);
|
||||
await _workerStream.WriteAsync(lengthPrefix, cancellationToken).ConfigureAwait(false);
|
||||
await _workerStream.WriteAsync(payload, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Writes an oversized frame header to the worker stream for testing frame size limits.</summary>
|
||||
/// <param name="payloadLength">Length of the oversized payload in bytes.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
public async Task WriteOversizedFrameHeaderAsync(
|
||||
uint payloadLength,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (payloadLength <= _frameOptions.MaxMessageBytes)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(payloadLength),
|
||||
payloadLength,
|
||||
"Payload length must exceed the configured maximum.");
|
||||
}
|
||||
|
||||
byte[] lengthPrefix = new byte[sizeof(uint)];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(lengthPrefix, payloadLength);
|
||||
await _workerStream.WriteAsync(lengthPrefix, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Disposes the worker-side stream.</summary>
|
||||
public async ValueTask DisposeWorkerSideAsync()
|
||||
{
|
||||
if (_workerSideDisposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _workerStream.DisposeAsync().ConfigureAwait(false);
|
||||
_workerSideDisposed = true;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await DisposeWorkerSideAsync().ConfigureAwait(false);
|
||||
if (_gatewayStream is not null)
|
||||
{
|
||||
await _gatewayStream.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private WorkerEnvelope CreateEnvelope(
|
||||
string correlationId,
|
||||
Action<WorkerEnvelope> setBody)
|
||||
{
|
||||
WorkerEnvelope envelope = new()
|
||||
{
|
||||
ProtocolVersion = _frameOptions.ProtocolVersion,
|
||||
SessionId = SessionId,
|
||||
Sequence = AdvanceSequence(),
|
||||
CorrelationId = correlationId,
|
||||
};
|
||||
setBody(envelope);
|
||||
|
||||
return envelope;
|
||||
}
|
||||
|
||||
private ulong AdvanceSequence()
|
||||
{
|
||||
return ++NextWorkerSequence;
|
||||
}
|
||||
|
||||
private static NamedPipeClientStream CreateWorkerStream(string pipeName)
|
||||
{
|
||||
return new NamedPipeClientStream(
|
||||
".",
|
||||
pipeName,
|
||||
PipeDirection.InOut,
|
||||
PipeOptions.Asynchronous);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Server-002 regression: per <c>gateway.md</c> the gateway must terminate
|
||||
/// orphaned worker processes on startup. These tests pin that the terminator
|
||||
/// kills leftover workers (matched by executable path, or by image name when
|
||||
/// the path is unreadable) without touching unrelated processes or itself.
|
||||
/// </summary>
|
||||
public sealed class OrphanWorkerTerminatorTests
|
||||
{
|
||||
private const string WorkerExecutablePath = @"C:\app\src\ZB.MOM.WW.MxGateway.Worker\bin\x86\Release\ZB.MOM.WW.MxGateway.Worker.exe";
|
||||
|
||||
[Fact]
|
||||
public void TerminateOrphans_KillsWorkerProcessesMatchingConfiguredExecutablePath()
|
||||
{
|
||||
FakeProcessInspector inspector = new(
|
||||
[
|
||||
new RunningProcessInfo(101, WorkerExecutablePath),
|
||||
new RunningProcessInfo(102, WorkerExecutablePath),
|
||||
]);
|
||||
OrphanWorkerTerminator terminator = CreateTerminator(inspector);
|
||||
|
||||
int killed = terminator.TerminateOrphans();
|
||||
|
||||
Assert.Equal(2, killed);
|
||||
Assert.Equal([101, 102], inspector.KilledProcessIds.Order());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TerminateOrphans_KillsImageNameMatchWhenExecutablePathUnreadable()
|
||||
{
|
||||
// The x64 gateway cannot introspect the x86 worker's main module, so the
|
||||
// path comes back null. Image-name match is the only signal — and it is
|
||||
// exactly the orphan worker case, so the process must still be killed.
|
||||
FakeProcessInspector inspector = new(
|
||||
[
|
||||
new RunningProcessInfo(201, ExecutablePath: null),
|
||||
]);
|
||||
OrphanWorkerTerminator terminator = CreateTerminator(inspector);
|
||||
|
||||
int killed = terminator.TerminateOrphans();
|
||||
|
||||
Assert.Equal(1, killed);
|
||||
Assert.Equal([201], inspector.KilledProcessIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TerminateOrphans_DoesNotKillUnrelatedProcessSharingImageName()
|
||||
{
|
||||
// A process with the same image name but a different executable path is
|
||||
// not our worker and must be left alone.
|
||||
FakeProcessInspector inspector = new(
|
||||
[
|
||||
new RunningProcessInfo(301, @"C:\other\place\ZB.MOM.WW.MxGateway.Worker.exe"),
|
||||
]);
|
||||
OrphanWorkerTerminator terminator = CreateTerminator(inspector);
|
||||
|
||||
int killed = terminator.TerminateOrphans();
|
||||
|
||||
Assert.Equal(0, killed);
|
||||
Assert.Empty(inspector.KilledProcessIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TerminateOrphans_DoesNotKillCurrentProcess()
|
||||
{
|
||||
FakeProcessInspector inspector = new(
|
||||
[
|
||||
new RunningProcessInfo(Environment.ProcessId, WorkerExecutablePath),
|
||||
]);
|
||||
OrphanWorkerTerminator terminator = CreateTerminator(inspector);
|
||||
|
||||
int killed = terminator.TerminateOrphans();
|
||||
|
||||
Assert.Equal(0, killed);
|
||||
Assert.Empty(inspector.KilledProcessIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TerminateOrphans_ContinuesWhenOneKillThrows()
|
||||
{
|
||||
FakeProcessInspector inspector = new(
|
||||
[
|
||||
new RunningProcessInfo(401, WorkerExecutablePath),
|
||||
new RunningProcessInfo(402, WorkerExecutablePath),
|
||||
])
|
||||
{
|
||||
ThrowOnKillProcessId = 401,
|
||||
};
|
||||
OrphanWorkerTerminator terminator = CreateTerminator(inspector);
|
||||
|
||||
int killed = terminator.TerminateOrphans();
|
||||
|
||||
Assert.Equal(1, killed);
|
||||
Assert.Contains(402, inspector.KilledProcessIds);
|
||||
}
|
||||
|
||||
private static OrphanWorkerTerminator CreateTerminator(IRunningProcessInspector inspector)
|
||||
{
|
||||
GatewayOptions options = new()
|
||||
{
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
ExecutablePath = WorkerExecutablePath,
|
||||
},
|
||||
};
|
||||
return new OrphanWorkerTerminator(
|
||||
Options.Create(options),
|
||||
inspector,
|
||||
new GatewayMetrics());
|
||||
}
|
||||
|
||||
private sealed class FakeProcessInspector(IReadOnlyList<RunningProcessInfo> processes)
|
||||
: IRunningProcessInspector
|
||||
{
|
||||
public List<int> KilledProcessIds { get; } = [];
|
||||
|
||||
public int? ThrowOnKillProcessId { get; init; }
|
||||
|
||||
public IReadOnlyList<RunningProcessInfo> GetProcessesByName(string processName) => processes;
|
||||
|
||||
public void Kill(int processId)
|
||||
{
|
||||
if (ThrowOnKillProcessId == processId)
|
||||
{
|
||||
throw new InvalidOperationException("Process has already exited.");
|
||||
}
|
||||
|
||||
KilledProcessIds.Add(processId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,801 @@
|
||||
using System.IO.Pipes;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
|
||||
|
||||
public sealed class WorkerClientTests
|
||||
{
|
||||
private const string SessionId = "session-worker-client";
|
||||
private const string Nonce = "nonce-worker-client";
|
||||
private const int WorkerProcessId = 4321;
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Verifies that StartAsync enters ready state after receiving worker hello and ready messages.</summary>
|
||||
[Fact]
|
||||
public async Task StartAsync_WithWorkerHelloAndReady_EntersReadyState()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(pipePair);
|
||||
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
Assert.Equal(WorkerProcessId, client.ProcessId);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that InvokeAsync completes a pending command when a matching reply arrives.</summary>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WithMatchingReply_CompletesPendingCommand()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(pipePair);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
TestTimeout,
|
||||
CancellationToken.None);
|
||||
|
||||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||||
Assert.False(string.IsNullOrWhiteSpace(commandEnvelope.CorrelationId));
|
||||
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateCommandReplyEnvelope(commandEnvelope.CorrelationId, MxCommandKind.Ping));
|
||||
|
||||
WorkerCommandReply reply = await invokeTask.WaitAsync(TestTimeout);
|
||||
|
||||
Assert.Equal(commandEnvelope.CorrelationId, reply.Reply.CorrelationId);
|
||||
Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that InvokeAsync ignores late replies and keeps the client ready.</summary>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WithLateReply_IgnoresLateReplyAndKeepsClientReady()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(pipePair);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
Task<WorkerCommandReply> timedOutInvokeTask = client.InvokeAsync(
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
TimeSpan.FromMilliseconds(50),
|
||||
CancellationToken.None);
|
||||
WorkerEnvelope timedOutCommand = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||||
|
||||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||||
async () => await timedOutInvokeTask);
|
||||
Assert.Equal(WorkerClientErrorCode.CommandTimeout, exception.ErrorCode);
|
||||
|
||||
// Send the stale reply for the already-timed-out command, then the second
|
||||
// command's reply. The pipe is FIFO, so the read loop processes (and discards)
|
||||
// the stale reply before the second reply — no fixed Task.Delay needed.
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateCommandReplyEnvelope(timedOutCommand.CorrelationId, MxCommandKind.Ping));
|
||||
|
||||
Task<WorkerCommandReply> secondInvokeTask = client.InvokeAsync(
|
||||
CreateCommand(MxCommandKind.GetWorkerInfo),
|
||||
TestTimeout,
|
||||
CancellationToken.None);
|
||||
WorkerEnvelope secondCommand = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateCommandReplyEnvelope(secondCommand.CorrelationId, MxCommandKind.GetWorkerInfo));
|
||||
|
||||
WorkerCommandReply reply = await secondInvokeTask.WaitAsync(TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
Assert.Equal(MxCommandKind.GetWorkerInfo, reply.Reply.Kind);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that ReadEventsAsync yields events in pipe order from the worker.</summary>
|
||||
[Fact]
|
||||
public async Task ReadEventsAsync_WithWorkerEvents_YieldsEventsInPipeOrder()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(pipePair);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
|
||||
|
||||
await using IAsyncEnumerator<WorkerEvent> events =
|
||||
client.ReadEventsAsync(cancellationTokenSource.Token).GetAsyncEnumerator(cancellationTokenSource.Token);
|
||||
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateEventEnvelope(sequence: 11, MxEventFamily.OnDataChange));
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateEventEnvelope(sequence: 12, MxEventFamily.OperationComplete));
|
||||
|
||||
Assert.True(await events.MoveNextAsync());
|
||||
Assert.Equal((ulong)11, events.Current.Event.WorkerSequence);
|
||||
Assert.Equal(MxEventFamily.OnDataChange, events.Current.Event.Family);
|
||||
|
||||
Assert.True(await events.MoveNextAsync());
|
||||
Assert.Equal((ulong)12, events.Current.Event.WorkerSequence);
|
||||
Assert.Equal(MxEventFamily.OperationComplete, events.Current.Event.Family);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the read loop faults the client when the event queue overflows.</summary>
|
||||
[Fact]
|
||||
public async Task ReadLoop_WhenEventQueueOverflows_FaultsClient()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(
|
||||
pipePair,
|
||||
new WorkerClientOptions
|
||||
{
|
||||
EventChannelCapacity = 1,
|
||||
EventChannelFullModeTimeout = TimeSpan.FromMilliseconds(50),
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||||
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
|
||||
});
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateEventEnvelope(sequence: 11, MxEventFamily.OnDataChange));
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateEventEnvelope(sequence: 12, MxEventFamily.OnDataChange));
|
||||
|
||||
await WaitUntilAsync(
|
||||
() => client.State == WorkerClientState.Faulted,
|
||||
TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that when the client faults it kills the owned worker process.
|
||||
/// The assertion waits on <see cref="FakeWorkerProcess.WaitForExitAsync"/>, which
|
||||
/// completes exactly when <c>Kill</c> runs, instead of polling <c>client.State</c>.
|
||||
/// Polling state is racy: <see cref="WorkerClient.SetFaulted"/> publishes the
|
||||
/// <c>Faulted</c> state before it calls <c>KillOwnedProcess</c>, so a state-based
|
||||
/// wait can observe <c>Faulted</c> while <c>KillCount</c> is still 0.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ReadLoop_WhenClientFaults_KillsOwnedWorkerProcess()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
FakeWorkerProcess process = new();
|
||||
await using WorkerClient client = CreateClient(
|
||||
pipePair,
|
||||
new WorkerClientOptions
|
||||
{
|
||||
EventChannelCapacity = 1,
|
||||
EventChannelFullModeTimeout = TimeSpan.FromMilliseconds(50),
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||||
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
|
||||
},
|
||||
processHandle: CreateProcessHandle(process));
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateEventEnvelope(sequence: 11, MxEventFamily.OnDataChange));
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateEventEnvelope(sequence: 12, MxEventFamily.OnDataChange));
|
||||
|
||||
// Deterministic: this completes the instant Kill() runs, with no timing window.
|
||||
using CancellationTokenSource exitTimeout = new(TestTimeout);
|
||||
await process.WaitForExitAsync(exitTimeout.Token);
|
||||
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
Assert.Equal(1, process.KillCount);
|
||||
Assert.True(process.KillEntireProcessTree);
|
||||
Assert.True(process.HasExited);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a worker faulting mid-command — the pipe dropping while an
|
||||
/// <see cref="WorkerClient.InvokeAsync"/> is still pending — completes the pending
|
||||
/// invoke task with a <see cref="WorkerClientException"/> carrying the
|
||||
/// pipe-disconnected error code rather than hanging until the command timeout.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WhenPipeDisconnectsMidCommand_FailsPendingInvokeWithPipeDisconnected()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(pipePair);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
TestTimeout,
|
||||
CancellationToken.None);
|
||||
|
||||
// The worker received the command but disconnects before replying.
|
||||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||||
await pipePair.DisposeWorkerSideAsync();
|
||||
|
||||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||||
async () => await invokeTask.WaitAsync(TestTimeout));
|
||||
|
||||
Assert.Equal(WorkerClientErrorCode.PipeDisconnected, exception.ErrorCode);
|
||||
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a worker emitting a <c>WorkerFault</c> envelope while an
|
||||
/// <see cref="WorkerClient.InvokeAsync"/> is pending completes the pending invoke
|
||||
/// task with a <see cref="WorkerClientException"/> carrying the worker-faulted
|
||||
/// error code.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WhenWorkerFaultsMidCommand_FailsPendingInvokeWithWorkerFaulted()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(pipePair);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
TestTimeout,
|
||||
CancellationToken.None);
|
||||
|
||||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||||
await pipePair.WorkerWriter.WriteAsync(CreateWorkerFaultEnvelope("scripted mid-command fault"));
|
||||
|
||||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||||
async () => await invokeTask.WaitAsync(TestTimeout));
|
||||
|
||||
Assert.Equal(WorkerClientErrorCode.WorkerFaulted, exception.ErrorCode);
|
||||
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadLoop_WhenPipeDisconnects_FaultsClient()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(pipePair);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
await pipePair.DisposeWorkerSideAsync();
|
||||
|
||||
await WaitUntilAsync(
|
||||
() => client.State == WorkerClientState.Faulted,
|
||||
TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the read loop stops the running worker metric when the pipe disconnects.</summary>
|
||||
[Fact]
|
||||
public async Task ReadLoop_WhenPipeDisconnects_StopsRunningWorkerMetric()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
using GatewayMetrics metrics = new();
|
||||
await using WorkerClient client = CreateClient(pipePair, metrics: metrics);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
Assert.Equal(1, metrics.GetSnapshot().WorkersRunning);
|
||||
|
||||
await pipePair.DisposeWorkerSideAsync();
|
||||
|
||||
await WaitUntilAsync(
|
||||
() => client.State == WorkerClientState.Faulted
|
||||
&& metrics.GetSnapshot().WorkersRunning == 0,
|
||||
TestTimeout);
|
||||
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(0, snapshot.WorkersRunning);
|
||||
Assert.Equal(1, snapshot.WorkerExits);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that DisposeAsync returns within a bounded timeout when the pipe read is blocked.</summary>
|
||||
[Fact]
|
||||
public async Task DisposeAsync_WhenPipeReadIsBlocked_ReturnsWithinBoundedTimeout()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
WorkerClient client = CreateClient(pipePair);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
DateTimeOffset startedAt = DateTimeOffset.UtcNow;
|
||||
await client.DisposeAsync().AsTask().WaitAsync(TestTimeout);
|
||||
TimeSpan elapsed = DateTimeOffset.UtcNow - startedAt;
|
||||
|
||||
Assert.True(
|
||||
elapsed < TimeSpan.FromSeconds(4),
|
||||
$"DisposeAsync took {elapsed.TotalMilliseconds:N0}ms.");
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the read loop updates the last heartbeat and worker process when a heartbeat arrives.</summary>
|
||||
[Fact]
|
||||
public async Task DisposeAsync_WhenOwnedWorkerStillRuns_KillsProcessBeforeDisposing()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
FakeWorkerProcess process = new();
|
||||
WorkerClient client = CreateClient(pipePair, processHandle: CreateProcessHandle(process));
|
||||
|
||||
await client.DisposeAsync().AsTask().WaitAsync(TestTimeout);
|
||||
|
||||
Assert.Equal(1, process.KillCount);
|
||||
Assert.True(process.KillEntireProcessTree);
|
||||
Assert.True(process.Disposed);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a heartbeat envelope updates the last-heartbeat timestamp and worker
|
||||
/// process id. Uses a <see cref="ManualTimeProvider"/> so the timestamp advance is
|
||||
/// deterministic instead of relying on a wall-clock <c>Task.Delay</c> exceeding
|
||||
/// <see cref="DateTimeOffset.UtcNow"/> resolution.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ReadLoop_WhenHeartbeatArrives_UpdatesLastHeartbeatAndWorkerProcess()
|
||||
{
|
||||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-18T12:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(pipePair, timeProvider: clock);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
DateTimeOffset previousHeartbeat = client.LastHeartbeatAt;
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
await pipePair.WorkerWriter.WriteAsync(CreateHeartbeatEnvelope(workerProcessId: 9876));
|
||||
|
||||
await WaitUntilAsync(
|
||||
() => client.ProcessId == 9876 && client.LastHeartbeatAt > previousHeartbeat,
|
||||
TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
Assert.Equal(previousHeartbeat + TimeSpan.FromSeconds(1), client.LastHeartbeatAt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the heartbeat monitor faults the client when the heartbeat expires.
|
||||
/// Uses an injected <see cref="ManualTimeProvider"/> so the grace comparison is deterministic
|
||||
/// instead of depending on real wall-clock advance; the monitor's
|
||||
/// <see cref="WorkerClientOptions.HeartbeatCheckInterval"/> timer stays on the real clock and
|
||||
/// observes the manually-advanced grace on its next tick.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task HeartbeatMonitor_WhenHeartbeatExpires_FaultsClient()
|
||||
{
|
||||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T12:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(
|
||||
pipePair,
|
||||
new WorkerClientOptions
|
||||
{
|
||||
HeartbeatGrace = TimeSpan.FromMilliseconds(80),
|
||||
HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
|
||||
EventChannelCapacity = 8,
|
||||
},
|
||||
timeProvider: clock);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(2));
|
||||
|
||||
await WaitUntilAsync(
|
||||
() => client.State == WorkerClientState.Faulted,
|
||||
TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Server-031 regression: while a command is in flight on the
|
||||
/// gateway↔worker pipe and the oldest pending command is younger
|
||||
/// than <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
|
||||
/// heartbeat watchdog must NOT fault on heartbeat-expired alone — the
|
||||
/// gap is more likely caused by pipe-write contention than by a hung
|
||||
/// worker. Mirrors Worker-023 on the worker side.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task HeartbeatMonitor_WhenCommandInFlightWithinCeiling_DoesNotFaultOnExpiredHeartbeat()
|
||||
{
|
||||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(
|
||||
pipePair,
|
||||
new WorkerClientOptions
|
||||
{
|
||||
HeartbeatGrace = TimeSpan.FromMilliseconds(80),
|
||||
HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
|
||||
EventChannelCapacity = 8,
|
||||
HeartbeatStuckCeiling = TimeSpan.FromSeconds(30),
|
||||
},
|
||||
timeProvider: clock);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
// Begin a command that the test never replies to — keeps the
|
||||
// PendingCommand alive in `_pendingCommands` for the duration.
|
||||
Task<WorkerCommandReply> pendingInvoke = client.InvokeAsync(
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
TestTimeout,
|
||||
CancellationToken.None);
|
||||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||||
|
||||
// Advance well past HeartbeatGrace but well within HeartbeatStuckCeiling.
|
||||
clock.Advance(TimeSpan.FromSeconds(2));
|
||||
|
||||
// Give the heartbeat monitor a few real check-intervals to observe the gap.
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(150));
|
||||
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
Assert.False(pendingInvoke.IsCompleted);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Server-031 regression: once the oldest pending command exceeds
|
||||
/// <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
|
||||
/// heartbeat watchdog fires anyway — a truly stuck COM call shouldn't
|
||||
/// keep the watchdog suppressed indefinitely.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task HeartbeatMonitor_WhenPendingCommandExceedsStuckCeiling_FaultsClient()
|
||||
{
|
||||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(
|
||||
pipePair,
|
||||
new WorkerClientOptions
|
||||
{
|
||||
HeartbeatGrace = TimeSpan.FromMilliseconds(80),
|
||||
HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
|
||||
EventChannelCapacity = 8,
|
||||
HeartbeatStuckCeiling = TimeSpan.FromMilliseconds(200),
|
||||
},
|
||||
timeProvider: clock);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
Task<WorkerCommandReply> pendingInvoke = client.InvokeAsync(
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
TestTimeout,
|
||||
CancellationToken.None);
|
||||
await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||||
|
||||
// Advance the clock past HeartbeatStuckCeiling. The worker pipe's
|
||||
// PendingCommand.StartTimestamp uses TimeProvider.GetTimestamp(), so the
|
||||
// ManualTimeProvider's GetElapsedTime sees the advanced gap.
|
||||
clock.Advance(TimeSpan.FromSeconds(2));
|
||||
|
||||
await WaitUntilAsync(
|
||||
() => client.State == WorkerClientState.Faulted,
|
||||
TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Server-032 regression: a transient burst that exceeds
|
||||
/// <see cref="WorkerClientOptions.EventChannelCapacity"/> must be
|
||||
/// absorbed for up to <see cref="WorkerClientOptions.EventChannelFullModeTimeout"/>
|
||||
/// (the channel is configured for <c>BoundedChannelFullMode.Wait</c>);
|
||||
/// only when the wait elapses without progress is the worker faulted,
|
||||
/// and the diagnostic must name the channel capacity, depth, and
|
||||
/// actionable remediation.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task EnqueueWorkerEvent_WhenChannelFullPastTimeout_FaultsWithRichDiagnostic()
|
||||
{
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
await using WorkerClient client = CreateClient(
|
||||
pipePair,
|
||||
new WorkerClientOptions
|
||||
{
|
||||
EventChannelCapacity = 4,
|
||||
EventChannelFullModeTimeout = TimeSpan.FromMilliseconds(100),
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||||
HeartbeatCheckInterval = TimeSpan.FromSeconds(1),
|
||||
});
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
// Fill the 4-slot channel and write exactly one more to force the
|
||||
// overflow path. The gateway never opens a StreamEvents consumer, so
|
||||
// the events stay buffered. Exactly five events are written: the
|
||||
// worker client faults while reading the fifth, after which its read
|
||||
// loop stops — a sixth event would never be drained and its pipe
|
||||
// write would block forever on a full OS pipe buffer.
|
||||
for (ulong sequence = 1; sequence <= 5; sequence++)
|
||||
{
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateEventEnvelope(sequence: sequence, MxEventFamily.OnDataChange));
|
||||
}
|
||||
|
||||
await WaitUntilAsync(
|
||||
() => client.State == WorkerClientState.Faulted,
|
||||
TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
|
||||
// Reading the events channel after fault throws the propagated
|
||||
// WorkerClientException carrying the rich diagnostic message. The
|
||||
// drain is bounded by TestTimeout so a regression that leaves the
|
||||
// channel uncompleted fails the test instead of hanging it.
|
||||
using CancellationTokenSource drainTimeout = new(TestTimeout);
|
||||
WorkerClientException fault = await Assert.ThrowsAsync<WorkerClientException>(async () =>
|
||||
{
|
||||
await foreach (WorkerEvent _ in client.ReadEventsAsync(drainTimeout.Token))
|
||||
{
|
||||
}
|
||||
});
|
||||
Assert.Contains("Worker event channel rejected", fault.Message);
|
||||
Assert.Contains("of 4 capacity", fault.Message);
|
||||
Assert.Contains("StreamEvents", fault.Message);
|
||||
Assert.Contains("MxGateway:Events:QueueCapacity", fault.Message);
|
||||
}
|
||||
|
||||
private static WorkerClient CreateClient(
|
||||
PipePair pipePair,
|
||||
WorkerClientOptions? options = null,
|
||||
GatewayMetrics? metrics = null,
|
||||
WorkerProcessHandle? processHandle = null,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
WorkerFrameProtocolOptions frameOptions = new(SessionId);
|
||||
WorkerClientConnection connection = new(
|
||||
SessionId,
|
||||
Nonce,
|
||||
pipePair.GatewayStream,
|
||||
frameOptions,
|
||||
processHandle);
|
||||
|
||||
return new WorkerClient(connection, options, metrics, timeProvider);
|
||||
}
|
||||
|
||||
private static WorkerProcessHandle CreateProcessHandle(FakeWorkerProcess process)
|
||||
{
|
||||
return new WorkerProcessHandle(
|
||||
process,
|
||||
new WorkerProcessCommandLine("ZB.MOM.WW.MxGateway.Worker.exe", []),
|
||||
DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static async Task CompleteHandshakeAsync(
|
||||
WorkerClient client,
|
||||
PipePair pipePair)
|
||||
{
|
||||
Task startTask = client.StartAsync(CancellationToken.None);
|
||||
|
||||
WorkerEnvelope gatewayHello = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, gatewayHello.BodyCase);
|
||||
Assert.Equal(Nonce, gatewayHello.GatewayHello.Nonce);
|
||||
Assert.Equal(GatewayContractInfo.WorkerProtocolVersion, gatewayHello.GatewayHello.SupportedProtocolVersion);
|
||||
|
||||
await pipePair.WorkerWriter.WriteAsync(CreateWorkerHelloEnvelope());
|
||||
await pipePair.WorkerWriter.WriteAsync(CreateWorkerReadyEnvelope());
|
||||
await startTask.WaitAsync(TestTimeout);
|
||||
}
|
||||
|
||||
private static WorkerCommand CreateCommand(MxCommandKind kind)
|
||||
{
|
||||
return new WorkerCommand
|
||||
{
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = kind,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateWorkerHelloEnvelope()
|
||||
{
|
||||
return CreateWorkerEnvelope(
|
||||
correlationId: string.Empty,
|
||||
sequence: 1,
|
||||
envelope => envelope.WorkerHello = new WorkerHello
|
||||
{
|
||||
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||||
Nonce = Nonce,
|
||||
WorkerProcessId = WorkerProcessId,
|
||||
WorkerVersion = "fake-worker",
|
||||
});
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateWorkerReadyEnvelope()
|
||||
{
|
||||
return CreateWorkerEnvelope(
|
||||
correlationId: string.Empty,
|
||||
sequence: 2,
|
||||
envelope => envelope.WorkerReady = new WorkerReady
|
||||
{
|
||||
WorkerProcessId = WorkerProcessId,
|
||||
MxaccessProgid = "LMXProxy.LMXProxyServer.1",
|
||||
MxaccessClsid = "{C30B52F5-2CB5-4760-AF0A-3A344A7EB5DC}",
|
||||
});
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateCommandReplyEnvelope(
|
||||
string correlationId,
|
||||
MxCommandKind kind)
|
||||
{
|
||||
return CreateWorkerEnvelope(
|
||||
correlationId,
|
||||
sequence: 10,
|
||||
envelope => envelope.WorkerCommandReply = new WorkerCommandReply
|
||||
{
|
||||
Reply = new MxCommandReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
CorrelationId = correlationId,
|
||||
Kind = kind,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateEventEnvelope(
|
||||
ulong sequence,
|
||||
MxEventFamily family)
|
||||
{
|
||||
return CreateWorkerEnvelope(
|
||||
correlationId: string.Empty,
|
||||
sequence,
|
||||
envelope => envelope.WorkerEvent = new WorkerEvent
|
||||
{
|
||||
Event = new MxEvent
|
||||
{
|
||||
SessionId = SessionId,
|
||||
Family = family,
|
||||
WorkerSequence = sequence,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateWorkerFaultEnvelope(string diagnosticMessage)
|
||||
{
|
||||
return CreateWorkerEnvelope(
|
||||
correlationId: string.Empty,
|
||||
sequence: 30,
|
||||
envelope => envelope.WorkerFault = new WorkerFault
|
||||
{
|
||||
Category = WorkerFaultCategory.MxaccessCommandFailed,
|
||||
DiagnosticMessage = diagnosticMessage,
|
||||
ProtocolStatus = new ProtocolStatus
|
||||
{
|
||||
Code = ProtocolStatusCode.WorkerUnavailable,
|
||||
Message = diagnosticMessage,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateHeartbeatEnvelope(int workerProcessId)
|
||||
{
|
||||
return CreateWorkerEnvelope(
|
||||
correlationId: string.Empty,
|
||||
sequence: 20,
|
||||
envelope => envelope.WorkerHeartbeat = new WorkerHeartbeat
|
||||
{
|
||||
WorkerProcessId = workerProcessId,
|
||||
State = WorkerState.Ready,
|
||||
LastStaActivityTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
|
||||
PendingCommandCount = 0,
|
||||
OutboundEventQueueDepth = 0,
|
||||
});
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateWorkerEnvelope(
|
||||
string correlationId,
|
||||
ulong sequence,
|
||||
Action<WorkerEnvelope> setBody)
|
||||
{
|
||||
WorkerEnvelope envelope = new()
|
||||
{
|
||||
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||||
SessionId = SessionId,
|
||||
Sequence = sequence,
|
||||
CorrelationId = correlationId,
|
||||
};
|
||||
setBody(envelope);
|
||||
|
||||
return envelope;
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(
|
||||
Func<bool> predicate,
|
||||
TimeSpan timeout)
|
||||
{
|
||||
using CancellationTokenSource cancellationTokenSource = new(timeout);
|
||||
while (!predicate())
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PipePair : IAsyncDisposable
|
||||
{
|
||||
private readonly NamedPipeClientStream _workerStream;
|
||||
private bool _workerSideDisposed;
|
||||
|
||||
private PipePair(
|
||||
NamedPipeServerStream gatewayStream,
|
||||
NamedPipeClientStream workerStream)
|
||||
{
|
||||
GatewayStream = gatewayStream;
|
||||
_workerStream = workerStream;
|
||||
WorkerReader = new WorkerFrameReader(_workerStream, new WorkerFrameProtocolOptions(SessionId));
|
||||
WorkerWriter = new WorkerFrameWriter(_workerStream, new WorkerFrameProtocolOptions(SessionId));
|
||||
}
|
||||
|
||||
/// <summary>The gateway side of the named pipe connection.</summary>
|
||||
public NamedPipeServerStream GatewayStream { get; }
|
||||
|
||||
/// <summary>Frame reader for worker messages.</summary>
|
||||
public WorkerFrameReader WorkerReader { get; }
|
||||
|
||||
/// <summary>Frame writer for worker messages.</summary>
|
||||
public WorkerFrameWriter WorkerWriter { get; }
|
||||
|
||||
/// <summary>Creates a connected pipe pair for testing.</summary>
|
||||
public static async Task<PipePair> CreateAsync()
|
||||
{
|
||||
string pipeName = $"mxaccessgw-workerclient-tests-{Guid.NewGuid():N}";
|
||||
NamedPipeServerStream gatewayStream = new(
|
||||
pipeName,
|
||||
PipeDirection.InOut,
|
||||
maxNumberOfServerInstances: 1,
|
||||
PipeTransmissionMode.Byte,
|
||||
PipeOptions.Asynchronous);
|
||||
NamedPipeClientStream workerStream = new(
|
||||
".",
|
||||
pipeName,
|
||||
PipeDirection.InOut,
|
||||
PipeOptions.Asynchronous);
|
||||
|
||||
Task waitForConnectionTask = gatewayStream.WaitForConnectionAsync();
|
||||
await workerStream.ConnectAsync();
|
||||
await waitForConnectionTask;
|
||||
|
||||
return new PipePair(gatewayStream, workerStream);
|
||||
}
|
||||
|
||||
/// <summary>Disposes the worker side of the pipe.</summary>
|
||||
public async ValueTask DisposeWorkerSideAsync()
|
||||
{
|
||||
if (_workerSideDisposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _workerStream.DisposeAsync();
|
||||
_workerSideDisposed = true;
|
||||
}
|
||||
|
||||
/// <summary>Disposes the duplex stream.</summary>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await DisposeWorkerSideAsync();
|
||||
await GatewayStream.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeWorkerProcess : IWorkerProcess
|
||||
{
|
||||
private readonly TaskCompletionSource _exited = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public int Id { get; } = WorkerProcessId;
|
||||
|
||||
public bool HasExited { get; private set; }
|
||||
|
||||
public int? ExitCode { get; private set; }
|
||||
|
||||
public int KillCount { get; private set; }
|
||||
|
||||
public bool KillEntireProcessTree { get; private set; }
|
||||
|
||||
public bool Disposed { get; private set; }
|
||||
|
||||
public ValueTask WaitForExitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return new ValueTask(_exited.Task.WaitAsync(cancellationToken));
|
||||
}
|
||||
|
||||
public void Kill(bool entireProcessTree)
|
||||
{
|
||||
KillCount++;
|
||||
KillEntireProcessTree = entireProcessTree;
|
||||
HasExited = true;
|
||||
ExitCode = -1;
|
||||
_exited.TrySetResult();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Disposed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
using System.Buffers.Binary;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Coverage for <see cref="WorkerExecutableValidator"/> PE-header architecture parsing
|
||||
/// (finding Server-013). The validator reads the DOS <c>MZ</c> stub, follows the PE
|
||||
/// header offset at <c>0x3c</c>, checks the <c>PE\0\0</c> signature, and compares the
|
||||
/// machine field against the required <see cref="WorkerArchitecture"/>.
|
||||
/// </summary>
|
||||
public sealed class WorkerExecutableValidatorTests : IDisposable
|
||||
{
|
||||
private const ushort ImageFileMachineI386 = 0x014c;
|
||||
private const ushort ImageFileMachineAmd64 = 0x8664;
|
||||
|
||||
private readonly List<string> _tempFiles = [];
|
||||
|
||||
[Fact]
|
||||
public void Validate_X86ExecutableMatchingRequiredArchitecture_DoesNotThrow()
|
||||
{
|
||||
string path = WritePeFile(ImageFileMachineI386);
|
||||
|
||||
WorkerExecutableValidator.Validate(path, WorkerArchitecture.X86);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_X64ExecutableMatchingRequiredArchitecture_DoesNotThrow()
|
||||
{
|
||||
string path = WritePeFile(ImageFileMachineAmd64);
|
||||
|
||||
WorkerExecutableValidator.Validate(path, WorkerArchitecture.X64);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_X64ExecutableWhenX86Required_ThrowsInvalidExecutable()
|
||||
{
|
||||
string path = WritePeFile(ImageFileMachineAmd64);
|
||||
|
||||
WorkerProcessLaunchException exception = Assert.Throws<WorkerProcessLaunchException>(
|
||||
() => WorkerExecutableValidator.Validate(path, WorkerArchitecture.X86));
|
||||
|
||||
Assert.Equal(WorkerProcessLaunchErrorCode.InvalidExecutable, exception.ErrorCode);
|
||||
Assert.Contains("architecture", exception.Message, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_X86ExecutableWhenX64Required_ThrowsInvalidExecutable()
|
||||
{
|
||||
string path = WritePeFile(ImageFileMachineI386);
|
||||
|
||||
WorkerProcessLaunchException exception = Assert.Throws<WorkerProcessLaunchException>(
|
||||
() => WorkerExecutableValidator.Validate(path, WorkerArchitecture.X64));
|
||||
|
||||
Assert.Equal(WorkerProcessLaunchErrorCode.InvalidExecutable, exception.ErrorCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_FileWithoutMzHeader_ThrowsInvalidExecutable()
|
||||
{
|
||||
byte[] bytes = new byte[0x80];
|
||||
// Leave the first two bytes as zero so the MZ signature check fails.
|
||||
string path = WriteTempFile(bytes);
|
||||
|
||||
WorkerProcessLaunchException exception = Assert.Throws<WorkerProcessLaunchException>(
|
||||
() => WorkerExecutableValidator.Validate(path, WorkerArchitecture.X86));
|
||||
|
||||
Assert.Equal(WorkerProcessLaunchErrorCode.InvalidExecutable, exception.ErrorCode);
|
||||
Assert.Contains("MZ", exception.Message, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_FileTooSmallForPeHeader_ThrowsInvalidExecutable()
|
||||
{
|
||||
string path = WriteTempFile([(byte)'M', (byte)'Z']);
|
||||
|
||||
WorkerProcessLaunchException exception = Assert.Throws<WorkerProcessLaunchException>(
|
||||
() => WorkerExecutableValidator.Validate(path, WorkerArchitecture.X86));
|
||||
|
||||
Assert.Equal(WorkerProcessLaunchErrorCode.InvalidExecutable, exception.ErrorCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_FileWithoutPeSignature_ThrowsInvalidExecutable()
|
||||
{
|
||||
// Build a valid MZ header pointing at a PE offset that holds a wrong signature.
|
||||
byte[] bytes = new byte[0x100];
|
||||
bytes[0] = (byte)'M';
|
||||
bytes[1] = (byte)'Z';
|
||||
BinaryPrimitives.WriteInt32LittleEndian(bytes.AsSpan(0x3c, sizeof(int)), 0x80);
|
||||
// PE region left as zeros — the "PE\0\0" signature check fails.
|
||||
string path = WriteTempFile(bytes);
|
||||
|
||||
WorkerProcessLaunchException exception = Assert.Throws<WorkerProcessLaunchException>(
|
||||
() => WorkerExecutableValidator.Validate(path, WorkerArchitecture.X86));
|
||||
|
||||
Assert.Equal(WorkerProcessLaunchErrorCode.InvalidExecutable, exception.ErrorCode);
|
||||
Assert.Contains("PE", exception.Message, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private string WritePeFile(ushort machine)
|
||||
{
|
||||
const int peHeaderOffset = 0x80;
|
||||
byte[] bytes = new byte[peHeaderOffset + 6];
|
||||
bytes[0] = (byte)'M';
|
||||
bytes[1] = (byte)'Z';
|
||||
BinaryPrimitives.WriteInt32LittleEndian(bytes.AsSpan(0x3c, sizeof(int)), peHeaderOffset);
|
||||
bytes[peHeaderOffset] = (byte)'P';
|
||||
bytes[peHeaderOffset + 1] = (byte)'E';
|
||||
bytes[peHeaderOffset + 2] = 0;
|
||||
bytes[peHeaderOffset + 3] = 0;
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(bytes.AsSpan(peHeaderOffset + 4, sizeof(ushort)), machine);
|
||||
return WriteTempFile(bytes);
|
||||
}
|
||||
|
||||
private string WriteTempFile(byte[] bytes)
|
||||
{
|
||||
string path = Path.Combine(Path.GetTempPath(), $"mxgw-pe-{Guid.NewGuid():N}.bin");
|
||||
File.WriteAllBytes(path, bytes);
|
||||
_tempFiles.Add(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (string path in _tempFiles)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Best-effort cleanup of the temp PE fixtures.
|
||||
}
|
||||
}
|
||||
|
||||
_tempFiles.Clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
using System.Buffers.Binary;
|
||||
using Google.Protobuf;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
|
||||
|
||||
public sealed class WorkerFrameProtocolTests
|
||||
{
|
||||
private const string SessionId = "session-1";
|
||||
|
||||
/// <summary>Verifies that writing and reading a valid envelope round-trips the frame correctly.</summary>
|
||||
[Fact]
|
||||
public async Task WriteAndReadAsync_WithValidEnvelope_RoundTripsFrame()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = new(SessionId);
|
||||
await using MemoryStream stream = new();
|
||||
WorkerEnvelope original = CreateEnvelope();
|
||||
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
await writer.WriteAsync(original);
|
||||
stream.Position = 0;
|
||||
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerEnvelope parsed = await reader.ReadAsync();
|
||||
|
||||
Assert.Equal(original, parsed);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that reading a frame with partial reads reassembles the frame correctly.</summary>
|
||||
[Fact]
|
||||
public async Task ReadAsync_WithPartialReads_ReassemblesFrame()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = new(SessionId);
|
||||
WorkerEnvelope original = CreateEnvelope();
|
||||
byte[] frame = CreateFrame(original);
|
||||
await using ChunkedReadStream stream = new(frame, chunkSize: 2);
|
||||
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerEnvelope parsed = await reader.ReadAsync();
|
||||
|
||||
Assert.Equal(original, parsed);
|
||||
Assert.True(stream.ReadCallCount > 2);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that reading a frame with zero length throws a malformed length exception.</summary>
|
||||
[Fact]
|
||||
public async Task ReadAsync_WithZeroLengthFrame_ThrowsMalformedLength()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = new(SessionId);
|
||||
await using MemoryStream stream = new(new byte[sizeof(uint)]);
|
||||
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerFrameProtocolException exception =
|
||||
await Assert.ThrowsAsync<WorkerFrameProtocolException>(
|
||||
async () => await reader.ReadAsync());
|
||||
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.MalformedLength, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that reading a frame with oversized length throws before allocating the payload.</summary>
|
||||
[Fact]
|
||||
public async Task ReadAsync_WithOversizedLength_ThrowsBeforePayloadAllocation()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = new(SessionId, GatewayContractInfo.WorkerProtocolVersion, maxMessageBytes: 16);
|
||||
byte[] lengthPrefix = new byte[sizeof(uint)];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(lengthPrefix, 17);
|
||||
await using MemoryStream stream = new(lengthPrefix);
|
||||
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerFrameProtocolException exception =
|
||||
await Assert.ThrowsAsync<WorkerFrameProtocolException>(
|
||||
async () => await reader.ReadAsync());
|
||||
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.MessageTooLarge, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that reading a frame with wrong protocol version throws a protocol version mismatch exception.</summary>
|
||||
[Fact]
|
||||
public async Task ReadAsync_WithWrongProtocolVersion_ThrowsProtocolVersionMismatch()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = new(SessionId);
|
||||
WorkerEnvelope envelope = CreateEnvelope();
|
||||
envelope.ProtocolVersion++;
|
||||
await using MemoryStream stream = new(CreateFrame(envelope));
|
||||
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerFrameProtocolException exception =
|
||||
await Assert.ThrowsAsync<WorkerFrameProtocolException>(
|
||||
async () => await reader.ReadAsync());
|
||||
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.ProtocolVersionMismatch, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that reading a frame with wrong session ID throws a session mismatch exception.</summary>
|
||||
[Fact]
|
||||
public async Task ReadAsync_WithWrongSessionId_ThrowsSessionMismatch()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = new(SessionId);
|
||||
WorkerEnvelope envelope = CreateEnvelope();
|
||||
envelope.SessionId = "different-session";
|
||||
await using MemoryStream stream = new(CreateFrame(envelope));
|
||||
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerFrameProtocolException exception =
|
||||
await Assert.ThrowsAsync<WorkerFrameProtocolException>(
|
||||
async () => await reader.ReadAsync());
|
||||
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.SessionMismatch, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that reading a frame with malformed payload throws an invalid envelope exception.</summary>
|
||||
[Fact]
|
||||
public async Task ReadAsync_WithMalformedPayload_ThrowsInvalidEnvelope()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = new(SessionId);
|
||||
byte[] frame = CreateFrame([0x80]);
|
||||
await using MemoryStream stream = new(frame);
|
||||
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerFrameProtocolException exception =
|
||||
await Assert.ThrowsAsync<WorkerFrameProtocolException>(
|
||||
async () => await reader.ReadAsync());
|
||||
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.InvalidEnvelope, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that reading a frame with missing envelope body throws an invalid envelope exception.</summary>
|
||||
[Fact]
|
||||
public async Task ReadAsync_WithMissingEnvelopeBody_ThrowsInvalidEnvelope()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = new(SessionId);
|
||||
WorkerEnvelope envelope = CreateEnvelope();
|
||||
envelope.ClearBody();
|
||||
await using MemoryStream stream = new(CreateFrame(envelope));
|
||||
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerFrameProtocolException exception =
|
||||
await Assert.ThrowsAsync<WorkerFrameProtocolException>(
|
||||
async () => await reader.ReadAsync());
|
||||
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.InvalidEnvelope, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that writing an oversized envelope throws a message too large exception.</summary>
|
||||
[Fact]
|
||||
public async Task WriteAsync_WithOversizedEnvelope_ThrowsMessageTooLarge()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = new(SessionId, GatewayContractInfo.WorkerProtocolVersion, maxMessageBytes: 8);
|
||||
await using MemoryStream stream = new();
|
||||
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
WorkerFrameProtocolException exception =
|
||||
await Assert.ThrowsAsync<WorkerFrameProtocolException>(
|
||||
async () => await writer.WriteAsync(CreateEnvelope()));
|
||||
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.MessageTooLarge, exception.ErrorCode);
|
||||
Assert.Equal(0, stream.Length);
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateEnvelope()
|
||||
{
|
||||
return new WorkerEnvelope
|
||||
{
|
||||
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||||
SessionId = SessionId,
|
||||
Sequence = 1,
|
||||
CorrelationId = "correlation-1",
|
||||
WorkerHello = new WorkerHello
|
||||
{
|
||||
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||||
Nonce = "nonce",
|
||||
WorkerProcessId = 1234,
|
||||
WorkerVersion = "test-worker",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static byte[] CreateFrame(IMessage message)
|
||||
{
|
||||
return CreateFrame(message.ToByteArray());
|
||||
}
|
||||
|
||||
private static byte[] CreateFrame(byte[] payload)
|
||||
{
|
||||
byte[] frame = new byte[sizeof(uint) + payload.Length];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(0, sizeof(uint)), (uint)payload.Length);
|
||||
payload.CopyTo(frame.AsSpan(sizeof(uint)));
|
||||
|
||||
return frame;
|
||||
}
|
||||
|
||||
private sealed class ChunkedReadStream : MemoryStream
|
||||
{
|
||||
private readonly int _chunkSize;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="ChunkedReadStream"/> class with chunked reads.</summary>
|
||||
/// <param name="buffer">The buffer containing data to read.</param>
|
||||
/// <param name="chunkSize">The maximum number of bytes to read per operation.</param>
|
||||
public ChunkedReadStream(
|
||||
byte[] buffer,
|
||||
int chunkSize)
|
||||
: base(buffer)
|
||||
{
|
||||
_chunkSize = chunkSize;
|
||||
}
|
||||
|
||||
/// <summary>Gets the number of read calls made to the stream.</summary>
|
||||
public int ReadCallCount { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ValueTask<int> ReadAsync(
|
||||
Memory<byte> buffer,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ReadCallCount++;
|
||||
int requestedCount = Math.Min(buffer.Length, _chunkSize);
|
||||
|
||||
return base.ReadAsync(buffer[..requestedCount], cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
|
||||
|
||||
public sealed class WorkerProcessLauncherTests
|
||||
{
|
||||
private const string SessionId = "session-1";
|
||||
private const string PipeName = "mxaccess-gateway-123-session-1";
|
||||
private const string Nonce = "super-secret-nonce";
|
||||
|
||||
/// <summary>Verifies that a valid worker executable starts with correct bootstrap arguments and nonce environment variable.</summary>
|
||||
[Fact]
|
||||
public async Task LaunchAsync_WithValidWorker_StartsProcessWithBootstrapArgumentsAndNonceEnvironment()
|
||||
{
|
||||
using TestDirectory directory = TestDirectory.Create();
|
||||
string executablePath = directory.CreateWorkerExecutable(machine: 0x014c);
|
||||
FakeWorkerProcess process = new(processId: 1234);
|
||||
FakePipeReservation pipeReservation = new();
|
||||
FakeWorkerProcessFactory processFactory = new(process);
|
||||
GatewayMetrics metrics = new();
|
||||
WorkerProcessLauncher launcher = CreateLauncher(executablePath, processFactory, new SucceedingStartupProbe(), metrics);
|
||||
|
||||
using WorkerProcessHandle handle = await launcher.LaunchAsync(CreateRequest(pipeReservation));
|
||||
|
||||
Assert.Equal(1234, handle.ProcessId);
|
||||
Assert.Same(process, handle.Process);
|
||||
Assert.NotNull(processFactory.LastStartInfo);
|
||||
Assert.Equal(Path.GetFullPath(executablePath), processFactory.LastStartInfo.FileName);
|
||||
Assert.False(processFactory.LastStartInfo.UseShellExecute);
|
||||
Assert.True(processFactory.LastStartInfo.CreateNoWindow);
|
||||
Assert.Equal(
|
||||
["--session-id", SessionId, "--pipe-name", PipeName, "--protocol-version", "1"],
|
||||
processFactory.LastStartInfo.ArgumentList);
|
||||
Assert.Equal(Nonce, processFactory.LastStartInfo.Environment[WorkerProcessLauncher.WorkerNonceEnvironmentVariableName]);
|
||||
Assert.Equal(
|
||||
"2000",
|
||||
processFactory.LastStartInfo.Environment[
|
||||
WorkerProcessLauncher.WorkerPipeConnectAttemptTimeoutEnvironmentVariableName]);
|
||||
Assert.DoesNotContain(Nonce, handle.CommandLine.ToString(), StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(Nonce, string.Join(" ", handle.CommandLine.Arguments), StringComparison.Ordinal);
|
||||
Assert.False(pipeReservation.DisposeCalled);
|
||||
Assert.Equal(0, metrics.GetSnapshot().WorkersRunning);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a failed startup probe kills and disposes the worker process.</summary>
|
||||
[Fact]
|
||||
public async Task LaunchAsync_WhenStartupProbeFails_KillsAndDisposesWorker()
|
||||
{
|
||||
using TestDirectory directory = TestDirectory.Create();
|
||||
string executablePath = directory.CreateWorkerExecutable(machine: 0x014c);
|
||||
FakeWorkerProcess process = new(processId: 1234);
|
||||
FakePipeReservation pipeReservation = new();
|
||||
GatewayMetrics metrics = new();
|
||||
WorkerProcessLauncher launcher = CreateLauncher(
|
||||
executablePath,
|
||||
new FakeWorkerProcessFactory(process),
|
||||
new FailingStartupProbe(),
|
||||
metrics);
|
||||
|
||||
WorkerProcessLaunchException exception =
|
||||
await Assert.ThrowsAsync<WorkerProcessLaunchException>(
|
||||
async () => await launcher.LaunchAsync(CreateRequest(pipeReservation)));
|
||||
|
||||
Assert.Equal(WorkerProcessLaunchErrorCode.StartupFailed, exception.ErrorCode);
|
||||
Assert.True(process.KillCalled);
|
||||
Assert.True(process.DisposeCalled);
|
||||
Assert.True(pipeReservation.DisposeCalled);
|
||||
Assert.Equal(1, metrics.GetSnapshot().WorkerKills);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that transient startup probe failures are retried without respawning the worker process.</summary>
|
||||
[Fact]
|
||||
public async Task LaunchAsync_WhenStartupProbeFailsTransiently_RetriesWithoutRespawningWorker()
|
||||
{
|
||||
using TestDirectory directory = TestDirectory.Create();
|
||||
string executablePath = directory.CreateWorkerExecutable(machine: 0x014c);
|
||||
FakeWorkerProcess process = new(processId: 1234);
|
||||
FakeWorkerProcessFactory processFactory = new(process);
|
||||
GatewayMetrics metrics = new();
|
||||
WorkerProcessLauncher launcher = CreateLauncher(
|
||||
executablePath,
|
||||
processFactory,
|
||||
new TransientStartupProbe(failuresBeforeSuccess: 1),
|
||||
metrics,
|
||||
startupProbeRetryAttempts: 2,
|
||||
startupProbeRetryDelayMilliseconds: 1);
|
||||
|
||||
using WorkerProcessHandle handle = await launcher.LaunchAsync(CreateRequest());
|
||||
|
||||
Assert.Same(process, handle.Process);
|
||||
Assert.Equal(1, processFactory.StartCount);
|
||||
Assert.False(process.KillCalled);
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snapshot.RetryAttempts);
|
||||
Assert.Equal(1, snapshot.RetryAttemptsByArea["worker_startup"]);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a startup probe timeout kills and disposes the worker process.</summary>
|
||||
[Fact]
|
||||
public async Task LaunchAsync_WhenStartupTimesOut_KillsAndDisposesWorker()
|
||||
{
|
||||
using TestDirectory directory = TestDirectory.Create();
|
||||
string executablePath = directory.CreateWorkerExecutable(machine: 0x014c);
|
||||
FakeWorkerProcess process = new(processId: 1234);
|
||||
GatewayMetrics metrics = new();
|
||||
WorkerProcessLauncher launcher = CreateLauncher(
|
||||
executablePath,
|
||||
new FakeWorkerProcessFactory(process),
|
||||
new WaitingStartupProbe(),
|
||||
metrics,
|
||||
startupTimeoutSeconds: 1);
|
||||
|
||||
WorkerProcessLaunchException exception =
|
||||
await Assert.ThrowsAsync<WorkerProcessLaunchException>(
|
||||
async () => await launcher.LaunchAsync(CreateRequest()));
|
||||
|
||||
Assert.Equal(WorkerProcessLaunchErrorCode.StartupTimeout, exception.ErrorCode);
|
||||
Assert.True(process.KillCalled);
|
||||
Assert.True(process.DisposeCalled);
|
||||
Assert.Equal(1, metrics.GetSnapshot().WorkerKills);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a missing worker executable fails before attempting to start the process.</summary>
|
||||
[Fact]
|
||||
public async Task LaunchAsync_WhenExecutableDoesNotExist_FailsBeforeStartingProcess()
|
||||
{
|
||||
using TestDirectory directory = TestDirectory.Create();
|
||||
string executablePath = Path.Combine(directory.Path, "missing-worker.exe");
|
||||
FakeWorkerProcessFactory processFactory = new(new FakeWorkerProcess(processId: 1234));
|
||||
WorkerProcessLauncher launcher = CreateLauncher(executablePath, processFactory, new SucceedingStartupProbe());
|
||||
|
||||
WorkerProcessLaunchException exception =
|
||||
await Assert.ThrowsAsync<WorkerProcessLaunchException>(
|
||||
async () => await launcher.LaunchAsync(CreateRequest()));
|
||||
|
||||
Assert.Equal(WorkerProcessLaunchErrorCode.ExecutableNotFound, exception.ErrorCode);
|
||||
Assert.Null(processFactory.LastStartInfo);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a worker executable with mismatched architecture fails before attempting to start.</summary>
|
||||
[Fact]
|
||||
public async Task LaunchAsync_WhenExecutableArchitectureDoesNotMatch_FailsBeforeStartingProcess()
|
||||
{
|
||||
using TestDirectory directory = TestDirectory.Create();
|
||||
string executablePath = directory.CreateWorkerExecutable(machine: 0x8664);
|
||||
FakeWorkerProcessFactory processFactory = new(new FakeWorkerProcess(processId: 1234));
|
||||
WorkerProcessLauncher launcher = CreateLauncher(executablePath, processFactory, new SucceedingStartupProbe());
|
||||
|
||||
WorkerProcessLaunchException exception =
|
||||
await Assert.ThrowsAsync<WorkerProcessLaunchException>(
|
||||
async () => await launcher.LaunchAsync(CreateRequest()));
|
||||
|
||||
Assert.Equal(WorkerProcessLaunchErrorCode.InvalidExecutable, exception.ErrorCode);
|
||||
Assert.Null(processFactory.LastStartInfo);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a worker that has already exited fails and disposes without additional killing.</summary>
|
||||
[Fact]
|
||||
public async Task LaunchAsync_WhenWorkerAlreadyExited_FailsAndDisposesWorkerWithoutKill()
|
||||
{
|
||||
using TestDirectory directory = TestDirectory.Create();
|
||||
string executablePath = directory.CreateWorkerExecutable(machine: 0x014c);
|
||||
FakeWorkerProcess process = new(processId: 1234)
|
||||
{
|
||||
HasExited = true,
|
||||
ExitCode = 42,
|
||||
};
|
||||
WorkerProcessLauncher launcher = CreateLauncher(
|
||||
executablePath,
|
||||
new FakeWorkerProcessFactory(process),
|
||||
new WorkerProcessStartedProbe());
|
||||
|
||||
WorkerProcessLaunchException exception =
|
||||
await Assert.ThrowsAsync<WorkerProcessLaunchException>(
|
||||
async () => await launcher.LaunchAsync(CreateRequest()));
|
||||
|
||||
Assert.Equal(WorkerProcessLaunchErrorCode.StartupFailed, exception.ErrorCode);
|
||||
Assert.False(process.KillCalled);
|
||||
Assert.True(process.DisposeCalled);
|
||||
}
|
||||
|
||||
private static WorkerProcessLauncher CreateLauncher(
|
||||
string executablePath,
|
||||
IWorkerProcessFactory processFactory,
|
||||
IWorkerStartupProbe startupProbe,
|
||||
GatewayMetrics? metrics = null,
|
||||
int startupTimeoutSeconds = 30,
|
||||
int startupProbeRetryAttempts = 3,
|
||||
int startupProbeRetryDelayMilliseconds = 250)
|
||||
{
|
||||
GatewayOptions options = new()
|
||||
{
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
ExecutablePath = executablePath,
|
||||
RequiredArchitecture = WorkerArchitecture.X86,
|
||||
StartupTimeoutSeconds = startupTimeoutSeconds,
|
||||
StartupProbeRetryAttempts = startupProbeRetryAttempts,
|
||||
StartupProbeRetryDelayMilliseconds = startupProbeRetryDelayMilliseconds,
|
||||
},
|
||||
};
|
||||
|
||||
return new WorkerProcessLauncher(
|
||||
Options.Create(options),
|
||||
processFactory,
|
||||
startupProbe,
|
||||
metrics ?? new GatewayMetrics());
|
||||
}
|
||||
|
||||
private static WorkerProcessLaunchRequest CreateRequest(IDisposable? pipeReservation = null)
|
||||
{
|
||||
return new WorkerProcessLaunchRequest(
|
||||
SessionId,
|
||||
PipeName,
|
||||
GatewayContractInfo.WorkerProtocolVersion,
|
||||
Nonce,
|
||||
pipeReservation);
|
||||
}
|
||||
|
||||
/// <summary>Fake worker process factory for testing process launch logic.</summary>
|
||||
private sealed class FakeWorkerProcessFactory(IWorkerProcess process) : IWorkerProcessFactory
|
||||
{
|
||||
/// <summary>Gets the most recent process start information.</summary>
|
||||
public ProcessStartInfo? LastStartInfo { get; private set; }
|
||||
|
||||
/// <summary>Gets the number of times the process factory has started a process.</summary>
|
||||
public int StartCount { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public IWorkerProcess Start(ProcessStartInfo startInfo)
|
||||
{
|
||||
StartCount++;
|
||||
LastStartInfo = startInfo;
|
||||
return process;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake worker process for testing process lifecycle and exit behavior.</summary>
|
||||
private sealed class FakeWorkerProcess(int processId) : IWorkerProcess
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public int Id { get; } = processId;
|
||||
|
||||
/// <summary>Gets or sets a value indicating whether the process has exited.</summary>
|
||||
public bool HasExited { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the process exit code.</summary>
|
||||
public int? ExitCode { get; set; }
|
||||
|
||||
/// <summary>Gets a value indicating whether the Dispose method was called.</summary>
|
||||
public bool DisposeCalled { get; private set; }
|
||||
|
||||
/// <summary>Gets a value indicating whether the Kill method was called.</summary>
|
||||
public bool KillCalled { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask WaitForExitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Kill(bool entireProcessTree)
|
||||
{
|
||||
Assert.True(entireProcessTree);
|
||||
KillCalled = true;
|
||||
HasExited = true;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
DisposeCalled = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake startup probe that immediately succeeds.</summary>
|
||||
private sealed class SucceedingStartupProbe : IWorkerStartupProbe
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task WaitUntilReadyAsync(
|
||||
IWorkerProcess process,
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake startup probe that always fails.</summary>
|
||||
private sealed class FailingStartupProbe : IWorkerStartupProbe
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task WaitUntilReadyAsync(
|
||||
IWorkerProcess process,
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
throw new InvalidOperationException("Fake worker startup failed.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake startup probe that waits indefinitely to simulate a startup timeout.</summary>
|
||||
private sealed class WaitingStartupProbe : IWorkerStartupProbe
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public async Task WaitUntilReadyAsync(
|
||||
IWorkerProcess process,
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake startup probe that fails a configurable number of times before succeeding.</summary>
|
||||
private sealed class TransientStartupProbe(int failuresBeforeSuccess) : IWorkerStartupProbe
|
||||
{
|
||||
private int _attempts;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task WaitUntilReadyAsync(
|
||||
IWorkerProcess process,
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (Interlocked.Increment(ref _attempts) <= failuresBeforeSuccess)
|
||||
{
|
||||
throw new IOException("The worker pipe was not ready yet.");
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fake pipe reservation for testing pipe lifecycle.</summary>
|
||||
private sealed class FakePipeReservation : IDisposable
|
||||
{
|
||||
/// <summary>Gets a value indicating whether the Dispose method was called.</summary>
|
||||
public bool DisposeCalled { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
DisposeCalled = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test helper that creates and cleans up a temporary directory for worker executable tests.</summary>
|
||||
private sealed class TestDirectory : IDisposable
|
||||
{
|
||||
private TestDirectory(string path)
|
||||
{
|
||||
Path = path;
|
||||
}
|
||||
|
||||
/// <summary>Gets the path to the temporary test directory.</summary>
|
||||
public string Path { get; }
|
||||
|
||||
/// <summary>Creates a new temporary directory for testing.</summary>
|
||||
public static TestDirectory Create()
|
||||
{
|
||||
string path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"mxgateway-tests-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(path);
|
||||
|
||||
return new TestDirectory(path);
|
||||
}
|
||||
|
||||
/// <summary>Creates a fake PE executable with the specified machine architecture for testing.</summary>
|
||||
/// <param name="machine">PE machine type constant (0x014c for x86, 0x8664 for x64).</param>
|
||||
/// <returns>Full path to the created executable file.</returns>
|
||||
public string CreateWorkerExecutable(ushort machine)
|
||||
{
|
||||
string path = System.IO.Path.Combine(Path, "ZB.MOM.WW.MxGateway.Worker.exe");
|
||||
byte[] bytes = new byte[0x100];
|
||||
bytes[0] = (byte)'M';
|
||||
bytes[1] = (byte)'Z';
|
||||
BitConverter.GetBytes(0x80).CopyTo(bytes, 0x3c);
|
||||
bytes[0x80] = (byte)'P';
|
||||
bytes[0x81] = (byte)'E';
|
||||
bytes[0x82] = 0;
|
||||
bytes[0x83] = 0;
|
||||
BitConverter.GetBytes(machine).CopyTo(bytes, 0x84);
|
||||
File.WriteAllBytes(path, bytes);
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
Directory.Delete(Path, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user