ci: make the windev CI lock exclusive, stop the scheduled double run, isolate test secrets stores (mxaccessgw#150)
Diagnosis of the retained Windows-nightly failures (119 scheduled nightlies, 28 red; job logs from the Gitea API, C:\build\mxaccessgw-ci\ci.log and the windev System log, read-only): - Checkout (#148, run 827, 2026-09-22): the windev "atomic mkdir" lock was not exclusive. New-Item -ItemType Directory checks for the directory and then calls CreateDirectory, which succeeds on an existing directory; a pwsh race harness had both contenders take it in 1900 of 2000 simultaneous rounds. A scheduled run starts windows-x86 and nightly-windev in the same second, both bootstraps got in, and the nightly's checkout died on .git/index.lock. Fix: the bootstrap (and a standalone windev-worker-ci.ps1) holds an exclusive FileShare.None handle on C:\build\mxaccessgw-ci.ci-lock instead (0 of 1000 double acquisitions in the same harness). The OS drops it if the holder dies, so a killed run leaves no stale lock, and a holder removes a leftover .git\index.lock, which can only be debris from a killed git. - windows-x86 no longer runs on the schedule: nightly-windev's live mode already runs the same Worker build + tests first, so the scheduled copy only doubled the load and created the race. - Host resources (#128/#129/#131-#135/#138, 2026-07-24..08-03): windev's System log has Resource-Exhaustion-Detector event 2004 (low virtual memory) every few minutes from 2026-07-23 until an unexpected shutdown/hard reset on 2026-08-03; OOM in MSBuild/csc/protoc, csc 0xC0000142, PowerShell dying before its first log line and SSH kex resets all fall in that window and stop after the reset. Host-side; windev-worker-ci.ps1 now logs free commit, physical memory, disk and testhost count at start and on failure so the next one is correlatable. - SSH "No route to host" (#144, 2026-09-16): windev was down (unexpected shutdown 00:58, back 05:50). - Dashboard host build (#143, run 817): two host-building test classes failed at the same instant in SqliteSecretsStoreMigrator with "cannot start a transaction within a transaction" -- parallel classes migrating the one per-process secrets store. Tests now build the gateway through TestSupport/TestGatewayHost, which gives each host its own store; TestGatewayHostTests guards it. - Gateway E2E "Worker pipe disconnected" (#142/#145/#146/#147) and handshaking-to-ready (#141) were the fake-worker partial-frame cancellation and readiness-race defects fixed in #153 (d779daf6): 7 gateway-suite failures in 51 windev runs before it, 0 in 28 since. The Worker watchdog and burst-coalescing failures (#126/#127/#139) last failed before50322bac/7da52b65(Aug 2026). - 2026-10-15..20 was the NU1903 package-audit restore failure, fixed infb20997. Docs: GatewayTesting.md (lock, host-resource triage) and scripts/ci/README.md. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -11,6 +11,7 @@ using ZB.MOM.WW.MxGateway.Server;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
using AuthenticationOptions = ZB.MOM.WW.MxGateway.Server.Configuration.AuthenticationOptions;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
@@ -115,7 +116,7 @@ public sealed class AlarmFeedAuthorizationTests
|
||||
[Fact]
|
||||
public async Task AlarmsPolicy_IsRegisteredWithNonBypassableRequirement()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
await using WebApplication app = TestGatewayHost.Build([]);
|
||||
IAuthorizationPolicyProvider provider = app.Services
|
||||
.GetRequiredService<IAuthorizationPolicyProvider>();
|
||||
|
||||
@@ -147,7 +148,7 @@ public sealed class AlarmFeedAuthorizationTests
|
||||
|
||||
Assert.Equal(DashboardAuthenticationDefaults.AlarmsPolicy, hubAttribute?.Policy);
|
||||
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
await using WebApplication app = TestGatewayHost.Build([]);
|
||||
RouteEndpoint alarmsPage = ((IEndpointRouteBuilder)app).DataSources
|
||||
.SelectMany(source => source.Endpoints)
|
||||
.OfType<RouteEndpoint>()
|
||||
|
||||
@@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
@@ -20,7 +21,7 @@ public sealed class DashboardCookieOptionsTests
|
||||
[Fact]
|
||||
public async Task Build_ConfiguresSecureDashboardCookie()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
await using WebApplication app = TestGatewayHost.Build([]);
|
||||
IOptionsMonitor<CookieAuthenticationOptions> optionsMonitor = app.Services
|
||||
.GetRequiredService<IOptionsMonitor<CookieAuthenticationOptions>>();
|
||||
|
||||
@@ -49,7 +50,7 @@ public sealed class DashboardCookieOptionsTests
|
||||
[Fact]
|
||||
public async Task Build_WithRequireHttpsCookieFalse_UsesSameAsRequestAndPlainName()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build(
|
||||
await using WebApplication app = TestGatewayHost.Build(
|
||||
["--MxGateway:Dashboard:RequireHttpsCookie=false"]);
|
||||
IOptionsMonitor<CookieAuthenticationOptions> optionsMonitor = app.Services
|
||||
.GetRequiredService<IOptionsMonitor<CookieAuthenticationOptions>>();
|
||||
@@ -71,7 +72,7 @@ public sealed class DashboardCookieOptionsTests
|
||||
[Fact]
|
||||
public async Task Build_WithCookieNameOverride_UsesConfiguredName()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build(
|
||||
await using WebApplication app = TestGatewayHost.Build(
|
||||
["--MxGateway:Dashboard:CookieName=MxGatewayDashboard.env2"]);
|
||||
IOptionsMonitor<CookieAuthenticationOptions> optionsMonitor = app.Services
|
||||
.GetRequiredService<IOptionsMonitor<CookieAuthenticationOptions>>();
|
||||
|
||||
@@ -6,6 +6,7 @@ using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.MxGateway.Server;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
@@ -16,7 +17,7 @@ public sealed class DashboardDisableLoginTests
|
||||
[Fact]
|
||||
public async Task DisableLoginOff_CookieSchemeUsesCookieHandler()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
await using WebApplication app = TestGatewayHost.Build([]);
|
||||
IAuthenticationSchemeProvider provider =
|
||||
app.Services.GetRequiredService<IAuthenticationSchemeProvider>();
|
||||
|
||||
@@ -32,7 +33,7 @@ public sealed class DashboardDisableLoginTests
|
||||
[Fact]
|
||||
public async Task DisableLoginOn_CookieSchemeUsesAutoLoginHandler()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build(
|
||||
await using WebApplication app = TestGatewayHost.Build(
|
||||
["--MxGateway:Dashboard:DisableLogin=true"]);
|
||||
IAuthenticationSchemeProvider provider =
|
||||
app.Services.GetRequiredService<IAuthenticationSchemeProvider>();
|
||||
@@ -49,7 +50,7 @@ public sealed class DashboardDisableLoginTests
|
||||
[Fact]
|
||||
public async Task DisableLoginOn_AutoLoginPrincipalSatisfiesAdminAndViewerPolicies()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build(
|
||||
await using WebApplication app = TestGatewayHost.Build(
|
||||
["--MxGateway:Dashboard:DisableLogin=true"]);
|
||||
IAuthorizationService authorization =
|
||||
app.Services.GetRequiredService<IAuthorizationService>();
|
||||
|
||||
@@ -7,6 +7,7 @@ using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
@@ -17,7 +18,7 @@ public sealed class DashboardHubsRegistrationTests
|
||||
[Fact]
|
||||
public async Task Build_WhenDashboardEnabled_MapsAllThreeHubsAndTokenEndpoint()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
await using WebApplication app = TestGatewayHost.Build([]);
|
||||
IReadOnlyList<RouteEndpoint> endpoints = ((IEndpointRouteBuilder)app).DataSources
|
||||
.SelectMany(source => source.Endpoints)
|
||||
.OfType<RouteEndpoint>()
|
||||
@@ -40,7 +41,7 @@ public sealed class DashboardHubsRegistrationTests
|
||||
[Fact]
|
||||
public async Task Build_WhenDashboardEnabled_RegistersHubTokenService()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
await using WebApplication app = TestGatewayHost.Build([]);
|
||||
|
||||
HubTokenService tokens = app.Services.GetRequiredService<HubTokenService>();
|
||||
Assert.NotNull(tokens);
|
||||
@@ -55,7 +56,7 @@ public sealed class DashboardHubsRegistrationTests
|
||||
[Fact]
|
||||
public async Task Build_WhenDashboardEnabled_ResolvesBothEventMirrorInterfacesToOneInstance()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
await using WebApplication app = TestGatewayHost.Build([]);
|
||||
|
||||
IDashboardEventBroadcaster broadcaster = app.Services.GetRequiredService<IDashboardEventBroadcaster>();
|
||||
IDashboardSessionEventSubscriber subscriber = app.Services
|
||||
@@ -74,7 +75,7 @@ public sealed class DashboardHubsRegistrationTests
|
||||
[Fact]
|
||||
public async Task Build_WhenDashboardEnabled_ShortensCircuitRetentionAndTracksConnection()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
await using WebApplication app = TestGatewayHost.Build([]);
|
||||
|
||||
CircuitOptions circuitOptions = app.Services.GetRequiredService<IOptions<CircuitOptions>>().Value;
|
||||
Assert.Equal(TimeSpan.FromSeconds(30), circuitOptions.DisconnectedCircuitRetentionPeriod);
|
||||
@@ -96,7 +97,7 @@ public sealed class DashboardHubsRegistrationTests
|
||||
[Fact]
|
||||
public async Task Build_WhenDashboardEnabled_SharesOneBrowseSearchIndex()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
await using WebApplication app = TestGatewayHost.Build([]);
|
||||
|
||||
Assert.Same(
|
||||
app.Services.GetRequiredService<DashboardBrowseSearchIndex>(),
|
||||
|
||||
Reference in New Issue
Block a user