From 8ff9f38bbdb97c66424f9f4f3dfc8b180ff89116 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 13 Oct 2026 09:56:37 -0400 Subject: [PATCH 1/2] Make bootstrap cancellation test deterministic and deduplicate boundary cases --- .../GatewayCeilingPagingTests.cs | 2 - .../CentralCommunicationActorTests.cs | 53 ++++++++++++++----- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayCeilingPagingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayCeilingPagingTests.cs index 17c5d312..374147a6 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayCeilingPagingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayCeilingPagingTests.cs @@ -197,8 +197,6 @@ public sealed class GatewayCeilingPagingTests : IDisposable [InlineData(L - 1, 5000u)] [InlineData(L, 5000u)] [InlineData(L + 1, 5000u)] - [InlineData(Ceiling - 17, 5000u)] - [InlineData(Ceiling - 16, 5000u)] [InlineData(Ceiling + 1, 5000u)] [InlineData(L - 2, 0u)] [InlineData(L - 1, 0u)] diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs index d4e8a1fb..9843c8fe 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs @@ -1,6 +1,9 @@ using System.Diagnostics.CodeAnalysis; using Akka.Actor; using Akka.Cluster.Tools.PublishSubscribe; +using Akka.Event; +using Akka.TestKit; +using Akka.TestKit.Xunit2; using Microsoft.EntityFrameworkCore; using Shouldly; using Xunit; @@ -680,7 +683,7 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase /// Counts contact loads and fails the first failFirst of them. private sealed class CountingDbFactory( - IDbContextFactory inner, int failFirst = 0) + IDbContextFactory inner, int failFirst = 0, Action? onLoad = null) : IDbContextFactory { private int _calls; @@ -689,7 +692,9 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase public OtOpcUaConfigDbContext CreateDbContext() { - if (Interlocked.Increment(ref _calls) <= failFirst) + var call = Interlocked.Increment(ref _calls); + onLoad?.Invoke(call); + if (call <= failFirst) throw new InvalidOperationException("ConfigDb unreachable at boot"); return inner.CreateDbContext(); } @@ -747,31 +752,51 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase [Fact] public void Bootstrap_retries_stop_once_the_gate_is_published() { - // The provider throws on PreStart (call 1) and on the first load's resolve (call 2), which - // schedules a 1 s retry. A manual refresh then publishes while that retry is still PENDING; - // stop-on-publish must cancel it. Without the cancel the timer fires, loads again and - // republishes (loads == 3, publications == 2). + // Freeze time only for this fact: the cluster harness and every other retry test retain + // their real scheduler. gRPC mode needs neither cluster membership nor pub-sub here. + using var local = new TestKit(@"akka { + loglevel = INFO + scheduler.implementation = ""Akka.TestKit.TestScheduler, Akka.TestKit"" + }"); + var scheduler = (TestScheduler)local.Sys.Scheduler; + var loads = local.CreateTestProbe(); + var barrier = local.CreateTestProbe(); + local.Sys.EventStream.Subscribe(barrier.Ref, typeof(UnhandledMessage)); var inner = NewInMemoryDbFactory(); SeedGrpcCluster(inner, "C1", ("host-a:4053", "host-a", 4056, true, false)); - var factory = new CountingDbFactory(inner); + var factory = new CountingDbFactory(inner, onLoad: n => loads.Ref.Tell(n)); var psk = new RecordingPskProvider(); var calls = 0; + // PreStart and the first contact load fail resolution. The latter registers a retry. IMeshPskProvider? PskFactory() => Interlocked.Increment(ref calls) <= 2 ? throw new InvalidOperationException("race") : psk; - var actor = Sys.ActorOf(CentralCommunicationActor.Props( + var actor = local.Sys.ActorOf(CentralCommunicationActor.Props( factory, GrpcMode(), () => null, () => new RecordingDialer(), PskFactory, bootstrapRetryInterval: TimeSpan.FromSeconds(1))); + // A mailbox barrier also waits for PreStart to register the zero-delay periodic tick. + var marker = new object(); + actor.Tell(marker); + barrier.ExpectMsg(TimeSpan.FromSeconds(5)).Message.ShouldBeSameAs(marker); + scheduler.Advance(TimeSpan.Zero); + loads.ExpectMsg(1, TimeSpan.FromSeconds(5)); + local.AwaitCondition(() => Volatile.Read(ref calls) == 2, TimeSpan.FromSeconds(5)); - // Positive control: the failed first resolve happened, so a retry is pending. - AwaitCondition(() => Volatile.Read(ref calls) >= 2, TimeSpan.FromSeconds(5)); - Thread.Sleep(100); - actor.Tell(new CentralCommunicationActor.RefreshContacts()); - AwaitCondition(() => psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(5)); + // This mailbox marker is handled after the failed ContactsLoaded handler returns. + // Unlike observing the provider call alone, it proves retry registration is complete. + actor.Tell(marker); + barrier.ExpectMsg(TimeSpan.FromSeconds(5)).Message.ShouldBeSameAs(marker); - Thread.Sleep(2500); + local.EventFilter.Info(contains: "Known-cluster gate published").ExpectOne( + TimeSpan.FromSeconds(5), () => actor.Tell(new CentralCommunicationActor.RefreshContacts())); + loads.ExpectMsg(2, TimeSpan.FromSeconds(5)); + // Cross the pending retry deadline, leaving the 60-second periodic tick untouched. + // LoadContactsFromDb uses Task.Run, so observe its boundary for the original absence + // budget rather than asserting before a mistakenly retained retry can start its read. + scheduler.Advance(TimeSpan.FromMilliseconds(2500)); + loads.ExpectNoMsg(TimeSpan.FromMilliseconds(2500)); factory.Calls.ShouldBe(2); psk.KnownClusterPublications.Count.ShouldBe(1); } -- 2.52.0 From 76bdca90e94e814e06ef10b78271e2b492b2ba32 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 13 Oct 2026 10:52:12 -0400 Subject: [PATCH 2/2] ci: bound build and test concurrency --- .github/workflows/v2-ci.yml | 39 ++++++++++++++++--------------------- scripts/ci/README.md | 30 ++++++++++++++++++++++++++++ scripts/ci/ci.runsettings | 12 ++++++++++++ 3 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 scripts/ci/README.md create mode 100644 scripts/ci/ci.runsettings diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index 0368f0a6..45abcef3 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -1,7 +1,6 @@ # CI for the v2 branch — runs on every push + PR to the v2-akka-fuse / master -# branches. Layered into two jobs: -# build dotnet restore + build (fast feedback on compile errors) -# unit-tests the WHOLE solution minus the env-gated tiers (self-maintaining) +# branches. One job restores and builds once, then tests the WHOLE solution +# minus the env-gated tiers (self-maintaining). # # Skips E2E (Category=E2E) and LiveIntegration (needs a real gateway/PLC/VPN) — both # are env-gated tiers with no hosted-runner fixtures. (The skip-gated *.IntegrationTests @@ -20,38 +19,33 @@ on: branches: [v2-akka-fuse, master] workflow_dispatch: {} +# #707: one active run per repository across push/PR triggers. Gitea 1.26 supports +# workflow concurrency; newer pending runs can supersede an older pending run. +concurrency: + group: lmxopcua-v2-ci + cancel-in-progress: false + env: DOTNET_NOLOGO: "1" DOTNET_CLI_TELEMETRY_OPTOUT: "1" jobs: - build: + unit-tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-dotnet@v4 with: dotnet-version: 10.0.x - # #606: code-reviews/README.md must be regenerated from the per-module findings.md - # files; --check exits non-zero on drift without writing (see REVIEW-PROCESS.md section 5). + # #606: regenerated review summary must remain in sync with module findings. - name: code-reviews README drift check run: python3 code-reviews/regen-readme.py --check + # #707: bound project workers and avoid concurrent NuGet restore work. - name: dotnet restore - run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx + run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx --disable-parallel -m:2 - name: dotnet build - run: dotnet build ZB.MOM.WW.OtOpcUa.slnx --no-restore --configuration Release - - unit-tests: - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-dotnet@v4 - with: - dotnet-version: 10.0.x - - name: dotnet restore - run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx + run: dotnet build ZB.MOM.WW.OtOpcUa.slnx --no-restore --configuration Release -m:2 # #606: the runner's size is the context every timing-sensitive failure needs and the log never # recorded it (load average included: the two Gitea runners may be running jobs side by side). - name: runner capacity @@ -69,14 +63,15 @@ jobs: # are unreachable, so they run harmlessly here (and are counted by the skip-gate below). - name: dotnet test (whole solution, minus E2E + LiveIntegration) # #606: Akka TestKit time dilation for the Runtime + ControlPlane actor suites (read by their - # harnesses, tests/Server/*/Harness/TestTimeFactor.cs). This leg runs every test assembly at once - # on the runner, and their 2–8 s presence budgets — sized on an idle dev box — failed in a long + # harnesses, tests/Server/*/Harness/TestTimeFactor.cs). Their 2–8 s presence budgets + # — sized on an idle dev box — failed in a long # tail, a different test per run. Dilation only stretches waits: a presence wait still fails if # the message never comes, and absence windows get longer (stricter). Local runs stay at 1. env: AKKA_TEST_TIMEFACTOR: "3" run: > - dotnet test ZB.MOM.WW.OtOpcUa.slnx --configuration Release --no-restore + dotnet test ZB.MOM.WW.OtOpcUa.slnx --configuration Release --no-build + -m:1 -p:TestTfmsInParallel=false --settings scripts/ci/ci.runsettings --filter "Category!=E2E&Category!=LiveIntegration" --logger "trx" --results-directory artifacts/trx # 07/S-2 fail-on-skip: a whole-solution run that executed ZERO tests is a masked outage diff --git a/scripts/ci/README.md b/scripts/ci/README.md new file mode 100644 index 00000000..c7f97d5c --- /dev/null +++ b/scripts/ci/README.md @@ -0,0 +1,30 @@ +# Whole-solution CI + +The `v2-ci` workflow restores, builds and tests in one job so the test step uses +that job's Release binaries. Restore disables parallel package work; restore and +build use at most two MSBuild workers. Testing uses one MSBuild worker, disables +parallel target-framework runs and loads `ci.runsettings`: one VSTest process and +two xUnit collection workers with conservative scheduling. Assemblies that disable +parallelization keep that setting. + +The workflow retains the E2E/LiveIntegration exclusions, includes SlowUnit, keeps +Akka's test time factor at three, and requires at least 12,000 executed tests. +Runner capacity and TRX files remain available for diagnosing failures. These +limits reduce contention; they do not prove every timing failure has one cause. +There are no automatic test retries or changed assertion deadlines. + +Gitea 1.26 supports the repository-wide `lmxopcua-v2-ci` concurrency group. Only +one run is active; an active run is not cancelled, but a newer pending run may +supersede an older pending run. The group does not serialize other repositories +using the same host. + +To reproduce the bounded test leg after restore and Release build: + +```bash +AKKA_TEST_TIMEFACTOR=3 dotnet test ZB.MOM.WW.OtOpcUa.slnx \ + --configuration Release --no-build -m:1 -p:TestTfmsInParallel=false \ + --settings scripts/ci/ci.runsettings \ + --filter "Category!=E2E&Category!=LiveIntegration" \ + --logger trx --results-directory artifacts/trx +MIN_EXECUTED=12000 bash scripts/ci/assert-not-all-skipped.sh artifacts/trx/*.trx +``` diff --git a/scripts/ci/ci.runsettings b/scripts/ci/ci.runsettings new file mode 100644 index 00000000..25f10d1e --- /dev/null +++ b/scripts/ci/ci.runsettings @@ -0,0 +1,12 @@ + + + + + 1 + + + + 2 + conservative + + -- 2.52.0