Merge pull request 'Bound CI concurrency and stabilize test fixtures' (#711) from fix/bootstrap-and-boundary-test-fixtures into master

.github/workflows/v2-ci.yml

@@ -1,7 +1,6 @@
 # CI for the v2 branch — runs on every push + PR to the v2-akka-fuse / master
-# branches. Layered into two jobs:
+# branches. One job restores and builds once, then tests the WHOLE solution
-#   build           dotnet restore + build (fast feedback on compile errors)
+# minus the env-gated tiers (self-maintaining).
-#   unit-tests      the WHOLE solution minus the env-gated tiers (self-maintaining)
 #
 # Skips E2E (Category=E2E) and LiveIntegration (needs a real gateway/PLC/VPN) — both
 # are env-gated tiers with no hosted-runner fixtures. (The skip-gated *.IntegrationTests
@@ -20,38 +19,33 @@ on:
     branches: [v2-akka-fuse, master]
   workflow_dispatch: {}
 
+# #707: one active run per repository across push/PR triggers. Gitea 1.26 supports
+# workflow concurrency; newer pending runs can supersede an older pending run.
+concurrency:
+  group: lmxopcua-v2-ci
+  cancel-in-progress: false
+
 env:
   DOTNET_NOLOGO: "1"
   DOTNET_CLI_TELEMETRY_OPTOUT: "1"
 
 jobs:
 
-  build:
+  unit-tests:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-dotnet@v4
         with:
           dotnet-version: 10.0.x
-      # #606: code-reviews/README.md must be regenerated from the per-module findings.md
+      # #606: regenerated review summary must remain in sync with module findings.
-      # files; --check exits non-zero on drift without writing (see REVIEW-PROCESS.md section 5).
       - name: code-reviews README drift check
         run: python3 code-reviews/regen-readme.py --check
+      # #707: bound project workers and avoid concurrent NuGet restore work.
       - name: dotnet restore
-        run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx
+        run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx --disable-parallel -m:2
       - name: dotnet build
-        run: dotnet build ZB.MOM.WW.OtOpcUa.slnx --no-restore --configuration Release
+        run: dotnet build ZB.MOM.WW.OtOpcUa.slnx --no-restore --configuration Release -m:2
-
-  unit-tests:
-    needs: build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-dotnet@v4
-        with:
-          dotnet-version: 10.0.x
-      - name: dotnet restore
-        run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx
       # #606: the runner's size is the context every timing-sensitive failure needs and the log never
       # recorded it (load average included: the two Gitea runners may be running jobs side by side).
       - name: runner capacity
@@ -69,14 +63,15 @@ jobs:
       # are unreachable, so they run harmlessly here (and are counted by the skip-gate below).
       - name: dotnet test (whole solution, minus E2E + LiveIntegration)
         # #606: Akka TestKit time dilation for the Runtime + ControlPlane actor suites (read by their
-        # harnesses, tests/Server/*/Harness/TestTimeFactor.cs). This leg runs every test assembly at once
+        # harnesses, tests/Server/*/Harness/TestTimeFactor.cs). Their 2–8 s presence budgets
-        # on the runner, and their 2–8 s presence budgets — sized on an idle dev box — failed in a long
+        # — sized on an idle dev box — failed in a long
         # tail, a different test per run. Dilation only stretches waits: a presence wait still fails if
         # the message never comes, and absence windows get longer (stricter). Local runs stay at 1.
         env:
           AKKA_TEST_TIMEFACTOR: "3"
         run: >
-          dotnet test ZB.MOM.WW.OtOpcUa.slnx --configuration Release --no-restore
+          dotnet test ZB.MOM.WW.OtOpcUa.slnx --configuration Release --no-build
+          -m:1 -p:TestTfmsInParallel=false --settings scripts/ci/ci.runsettings
           --filter "Category!=E2E&Category!=LiveIntegration"
           --logger "trx" --results-directory artifacts/trx
       # 07/S-2 fail-on-skip: a whole-solution run that executed ZERO tests is a masked outage

scripts/ci/README.md

@@ -0,0 +1,30 @@
+# Whole-solution CI
+
+The `v2-ci` workflow restores, builds and tests in one job so the test step uses
+that job's Release binaries. Restore disables parallel package work; restore and
+build use at most two MSBuild workers. Testing uses one MSBuild worker, disables
+parallel target-framework runs and loads `ci.runsettings`: one VSTest process and
+two xUnit collection workers with conservative scheduling. Assemblies that disable
+parallelization keep that setting.
+
+The workflow retains the E2E/LiveIntegration exclusions, includes SlowUnit, keeps
+Akka's test time factor at three, and requires at least 12,000 executed tests.
+Runner capacity and TRX files remain available for diagnosing failures. These
+limits reduce contention; they do not prove every timing failure has one cause.
+There are no automatic test retries or changed assertion deadlines.
+
+Gitea 1.26 supports the repository-wide `lmxopcua-v2-ci` concurrency group. Only
+one run is active; an active run is not cancelled, but a newer pending run may
+supersede an older pending run. The group does not serialize other repositories
+using the same host.
+
+To reproduce the bounded test leg after restore and Release build:
+
+```bash
+AKKA_TEST_TIMEFACTOR=3 dotnet test ZB.MOM.WW.OtOpcUa.slnx \
+  --configuration Release --no-build -m:1 -p:TestTfmsInParallel=false \
+  --settings scripts/ci/ci.runsettings \
+  --filter "Category!=E2E&Category!=LiveIntegration" \
+  --logger trx --results-directory artifacts/trx
+MIN_EXECUTED=12000 bash scripts/ci/assert-not-all-skipped.sh artifacts/trx/*.trx
+```

scripts/ci/ci.runsettings

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <!-- #707: project scheduling is also bounded by dotnet test -m:1. -->
+  <RunConfiguration>
+    <MaxCpuCount>1</MaxCpuCount>
+  </RunConfiguration>
+  <xUnit>
+    <!-- Preserve each assembly's parallelization opt-out; bound enabled collections. -->
+    <MaxParallelThreads>2</MaxParallelThreads>
+    <ParallelAlgorithm>conservative</ParallelAlgorithm>
+  </xUnit>
+</RunSettings>

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayCeilingPagingTests.cs

@@ -197,8 +197,6 @@ public sealed class GatewayCeilingPagingTests : IDisposable
     [InlineData(L - 1, 5000u)]
     [InlineData(L, 5000u)]
     [InlineData(L + 1, 5000u)]
-    [InlineData(Ceiling - 17, 5000u)]
-    [InlineData(Ceiling - 16, 5000u)]
     [InlineData(Ceiling + 1, 5000u)]
     [InlineData(L - 2, 0u)]
     [InlineData(L - 1, 0u)]

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs

@@ -1,6 +1,9 @@
 using System.Diagnostics.CodeAnalysis;
 using Akka.Actor;
 using Akka.Cluster.Tools.PublishSubscribe;
+using Akka.Event;
+using Akka.TestKit;
+using Akka.TestKit.Xunit2;
 using Microsoft.EntityFrameworkCore;
 using Shouldly;
 using Xunit;
@@ -680,7 +683,7 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
 
     /// <summary>Counts contact loads and fails the first <c>failFirst</c> of them.</summary>
     private sealed class CountingDbFactory(
-        IDbContextFactory<OtOpcUaConfigDbContext> inner, int failFirst = 0)
+        IDbContextFactory<OtOpcUaConfigDbContext> inner, int failFirst = 0, Action<int>? onLoad = null)
         : IDbContextFactory<OtOpcUaConfigDbContext>
     {
         private int _calls;
@@ -689,7 +692,9 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
 
         public OtOpcUaConfigDbContext CreateDbContext()
         {
-            if (Interlocked.Increment(ref _calls) <= failFirst)
+            var call = Interlocked.Increment(ref _calls);
+            onLoad?.Invoke(call);
+            if (call <= failFirst)
                 throw new InvalidOperationException("ConfigDb unreachable at boot");
             return inner.CreateDbContext();
         }
@@ -747,31 +752,51 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
     [Fact]
     public void Bootstrap_retries_stop_once_the_gate_is_published()
     {
-        // The provider throws on PreStart (call 1) and on the first load's resolve (call 2), which
+        // Freeze time only for this fact: the cluster harness and every other retry test retain
-        // schedules a 1 s retry. A manual refresh then publishes while that retry is still PENDING;
+        // their real scheduler. gRPC mode needs neither cluster membership nor pub-sub here.
-        // stop-on-publish must cancel it. Without the cancel the timer fires, loads again and
+        using var local = new TestKit(@"akka {
-        // republishes (loads == 3, publications == 2).
+            loglevel = INFO
+            scheduler.implementation = ""Akka.TestKit.TestScheduler, Akka.TestKit""
+        }");
+        var scheduler = (TestScheduler)local.Sys.Scheduler;
+        var loads = local.CreateTestProbe();
+        var barrier = local.CreateTestProbe();
+        local.Sys.EventStream.Subscribe(barrier.Ref, typeof(UnhandledMessage));
         var inner = NewInMemoryDbFactory();
         SeedGrpcCluster(inner, "C1", ("host-a:4053", "host-a", 4056, true, false));
-        var factory = new CountingDbFactory(inner);
+        var factory = new CountingDbFactory(inner, onLoad: n => loads.Ref.Tell(n));
         var psk = new RecordingPskProvider();
         var calls = 0;
 
+        // PreStart and the first contact load fail resolution. The latter registers a retry.
         IMeshPskProvider? PskFactory() =>
             Interlocked.Increment(ref calls) <= 2 ? throw new InvalidOperationException("race") : psk;
 
-        var actor = Sys.ActorOf(CentralCommunicationActor.Props(
+        var actor = local.Sys.ActorOf(CentralCommunicationActor.Props(
             factory, GrpcMode(), () => null, () => new RecordingDialer(), PskFactory,
             bootstrapRetryInterval: TimeSpan.FromSeconds(1)));
+        // A mailbox barrier also waits for PreStart to register the zero-delay periodic tick.
+        var marker = new object();
+        actor.Tell(marker);
+        barrier.ExpectMsg<UnhandledMessage>(TimeSpan.FromSeconds(5)).Message.ShouldBeSameAs(marker);
+        scheduler.Advance(TimeSpan.Zero);
+        loads.ExpectMsg(1, TimeSpan.FromSeconds(5));
+        local.AwaitCondition(() => Volatile.Read(ref calls) == 2, TimeSpan.FromSeconds(5));
 
-        // Positive control: the failed first resolve happened, so a retry is pending.
+        // This mailbox marker is handled after the failed ContactsLoaded handler returns.
-        AwaitCondition(() => Volatile.Read(ref calls) >= 2, TimeSpan.FromSeconds(5));
+        // Unlike observing the provider call alone, it proves retry registration is complete.
-        Thread.Sleep(100);
+        actor.Tell(marker);
-        actor.Tell(new CentralCommunicationActor.RefreshContacts());
+        barrier.ExpectMsg<UnhandledMessage>(TimeSpan.FromSeconds(5)).Message.ShouldBeSameAs(marker);
-        AwaitCondition(() => psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(5));
 
-        Thread.Sleep(2500);
+        local.EventFilter.Info(contains: "Known-cluster gate published").ExpectOne(
+            TimeSpan.FromSeconds(5), () => actor.Tell(new CentralCommunicationActor.RefreshContacts()));
+        loads.ExpectMsg(2, TimeSpan.FromSeconds(5));
 
+        // Cross the pending retry deadline, leaving the 60-second periodic tick untouched.
+        // LoadContactsFromDb uses Task.Run, so observe its boundary for the original absence
+        // budget rather than asserting before a mistakenly retained retry can start its read.
+        scheduler.Advance(TimeSpan.FromMilliseconds(2500));
+        loads.ExpectNoMsg(TimeSpan.FromMilliseconds(2500));
         factory.Calls.ShouldBe(2);
         psk.KnownClusterPublications.Count.ShouldBe(1);
     }