ci: count individual skipped TRX results

.github/workflows/v2-ci.yml

@@ -1,7 +1,6 @@
 # CI for the v2 branch — runs on every push + PR to the v2-akka-fuse / master
-# branches. Layered into two jobs:
-#   build           dotnet restore + build (fast feedback on compile errors)
-#   unit-tests      the WHOLE solution minus the env-gated tiers (self-maintaining)
+# branches. One job restores and builds once, then tests the WHOLE solution
+# minus the env-gated tiers (self-maintaining).
 #
 # Skips E2E (Category=E2E) and LiveIntegration (needs a real gateway/PLC/VPN) — both
 # are env-gated tiers with no hosted-runner fixtures. (The skip-gated *.IntegrationTests
@@ -20,38 +19,35 @@ on:
     branches: [v2-akka-fuse, master]
   workflow_dispatch: {}
 
+# #707: one active run per repository across push/PR triggers. Gitea 1.26 supports
+# workflow concurrency; newer pending runs can supersede an older pending run.
+concurrency:
+  group: lmxopcua-v2-ci
+  cancel-in-progress: false
+
 env:
   DOTNET_NOLOGO: "1"
   DOTNET_CLI_TELEMETRY_OPTOUT: "1"
 
 jobs:
 
-  build:
+  unit-tests:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-dotnet@v4
         with:
           dotnet-version: 10.0.x
-      # #606: code-reviews/README.md must be regenerated from the per-module findings.md
-      # files; --check exits non-zero on drift without writing (see REVIEW-PROCESS.md section 5).
+      # #606: regenerated review summary must remain in sync with module findings.
       - name: code-reviews README drift check
         run: python3 code-reviews/regen-readme.py --check
+      - name: TRX skip guard fixtures
+        run: python3 scripts/ci/test-assert-not-all-skipped.py
+      # #707: bound project workers and avoid concurrent NuGet restore work.
       - name: dotnet restore
-        run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx
+        run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx --disable-parallel -m:2
       - name: dotnet build
-        run: dotnet build ZB.MOM.WW.OtOpcUa.slnx --no-restore --configuration Release
-
-  unit-tests:
-    needs: build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-dotnet@v4
-        with:
-          dotnet-version: 10.0.x
-      - name: dotnet restore
-        run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx
+        run: dotnet build ZB.MOM.WW.OtOpcUa.slnx --no-restore --configuration Release -m:2
       # #606: the runner's size is the context every timing-sensitive failure needs and the log never
       # recorded it (load average included: the two Gitea runners may be running jobs side by side).
       - name: runner capacity
@@ -69,14 +65,15 @@ jobs:
       # are unreachable, so they run harmlessly here (and are counted by the skip-gate below).
       - name: dotnet test (whole solution, minus E2E + LiveIntegration)
         # #606: Akka TestKit time dilation for the Runtime + ControlPlane actor suites (read by their
-        # harnesses, tests/Server/*/Harness/TestTimeFactor.cs). This leg runs every test assembly at once
-        # on the runner, and their 2–8 s presence budgets — sized on an idle dev box — failed in a long
+        # harnesses, tests/Server/*/Harness/TestTimeFactor.cs). Their 2–8 s presence budgets
+        # — sized on an idle dev box — failed in a long
         # tail, a different test per run. Dilation only stretches waits: a presence wait still fails if
         # the message never comes, and absence windows get longer (stricter). Local runs stay at 1.
         env:
           AKKA_TEST_TIMEFACTOR: "3"
         run: >
-          dotnet test ZB.MOM.WW.OtOpcUa.slnx --configuration Release --no-restore
+          dotnet test ZB.MOM.WW.OtOpcUa.slnx --configuration Release --no-build
+          -m:1 -p:TestTfmsInParallel=false --settings scripts/ci/ci.runsettings
           --filter "Category!=E2E&Category!=LiveIntegration"
           --logger "trx" --results-directory artifacts/trx
       # 07/S-2 fail-on-skip: a whole-solution run that executed ZERO tests is a masked outage

docs/AlarmTracking.md

@@ -456,9 +456,15 @@ from the scripted `AlarmCommandRouter`:
    carrying the authored `FullName` as the `ConditionId` and the
    authenticated `OperatorUser`. The driver forwards this to the Galaxy
    gateway → AVEVA via `GatewayGalaxyAlarmAcknowledger`.
-   **Fire-and-forget** — a failed upstream ack is not surfaced back to the
-   OPC UA client (mirrors the Galaxy write-outcome limitation; the local
-   `AlarmConditionState` SDK update already committed at step 1).
+   The Galaxy adapter awaits the correlated `AcknowledgeAlarmAsync` RPC:
+   gateway protocol failures throw, while a non-zero native return code is
+   warning-logged. **The actor route back to OPC UA is fire-and-forget** —
+   `DriverInstanceActor` logs and drops failures without replying to the
+   OPC UA client, whose local `AlarmConditionState` SDK update already
+   committed at step 1. This is a native-ack routing limitation; Galaxy
+   value writes have a separate correlated write-outcome path. An awaited
+   ack RPC does not prove the ack reached AVEVA event history: the H6 live
+   verification remains [operator-gated](plans/2026-06-15-stillpending-phase-3-opcua-standards-design.md).
 
 Only the **Acknowledge** is routed to the driver. **A native condition
 supports Acknowledge and AddComment only** (Gitea #587 review): `Confirm`,

docs/ScriptedAlarms.md

@@ -325,8 +325,10 @@ routes through a separate `NativeAlarmAckRouter` seam (instead of the scripted
 FullName)` inverse map, Primary-gated) → the owning driver's
 `IAlarmSource.AcknowledgeAsync` → Galaxy gateway → AVEVA. The call carries the
 authenticated operator's display name via the `AlarmAcknowledgeRequest.OperatorUser`
-field. This is fire-and-forget — a failed upstream ack is not surfaced back to the
-OPC UA client (mirrors the Galaxy write-outcome limitation). See [AlarmTracking.md
+field. The actor route back to OPC UA is fire-and-forget — a failed upstream ack
+is logged rather than surfaced back to the OPC UA client. The Galaxy adapter
+awaits its correlated RPC; this does not establish delivery to AVEVA event history.
+Galaxy value writes use a separate correlated write-outcome path. See [AlarmTracking.md
 §Native alarm acknowledge → AVEVA](AlarmTracking.md#native-alarm-acknowledge--aveva)
 for the full routing diagram.
 

docs/plans/2026-06-15-stillpending-phase-3-opcua-standards-design.md

@@ -120,7 +120,9 @@ OPC UA client Acknowledge on a NATIVE condition
   under Lock, like `AlarmCommandRouter`).
 - **Driver faulted / reconnecting** → `AcknowledgeAsync` fails or no-ops per the driver; the local condition
   state still reflects the SDK-applied ack (the OPC UA client sees Good). Surfacing a *failed* upstream ack
-  back to the client is out of scope (mirrors the Galaxy write-outcome fire-and-forget limitation).
+  back to the client is out of scope for this native-ack actor route. **Current clarification
+  (2026-10-13):** the Galaxy adapter awaits a correlated ack RPC, but the actor route does not
+  return its outcome to OPC UA; Galaxy value writes have a separate correlated write-outcome path.
 - **H4 native Enable/Disable** → `BadNotSupported` (decision #2).
 - **H2-bit:** no runtime behavior change for existing grants except closing the HistoryRead⇒HistoryUpdate
   authorization hole; the service itself stays rejected.

scripts/ci/README.md

@@ -0,0 +1,45 @@
+# Whole-solution CI
+
+The `v2-ci` workflow restores, builds and tests in one job so the test step uses
+that job's Release binaries. Restore disables parallel package work; restore and
+build use at most two MSBuild workers. Testing uses one MSBuild worker, disables
+parallel target-framework runs and loads `ci.runsettings`: one VSTest process and
+two xUnit collection workers with conservative scheduling. Assemblies that disable
+parallelization keep that setting.
+
+The workflow retains the E2E/LiveIntegration exclusions, includes SlowUnit, keeps
+Akka's test time factor at three, and requires at least 12,000 executed tests.
+Runner capacity and TRX files remain available for diagnosing failures. These
+limits reduce contention; they do not prove every timing failure has one cause.
+There are no automatic test retries or changed assertion deadlines.
+
+Gitea 1.26 supports the repository-wide `lmxopcua-v2-ci` concurrency group. Only
+one run is active; an active run is not cancelled, but a newer pending run may
+supersede an older pending run. The group does not serialize other repositories
+using the same host.
+
+To reproduce the bounded test leg after restore and Release build:
+
+```bash
+AKKA_TEST_TIMEFACTOR=3 dotnet test ZB.MOM.WW.OtOpcUa.slnx \
+  --configuration Release --no-build -m:1 -p:TestTfmsInParallel=false \
+  --settings scripts/ci/ci.runsettings \
+  --filter "Category!=E2E&Category!=LiveIntegration" \
+  --logger trx --results-directory artifacts/trx
+MIN_EXECUTED=12000 bash scripts/ci/assert-not-all-skipped.sh artifacts/trx/*.trx
+```
+
+The guard uses the existing TRX `Counters/@executed` value for the aggregate
+floor. Its `skipped` diagnostic counts individual `Results/UnitTestResult`
+records whose outcome is `NotExecuted`, using the document's XML namespace.
+Some VSTest reports set summary `notExecuted` to zero even when these records
+are present. Skip records are counted per file, including repeated runs; they
+are not a count of unique test methods and do not affect the executed floor.
+For malformed XML, the diagnostic retains the old summary-counter fallback;
+this guard does not add XML validation to its existing gate policy.
+
+Run the deterministic script fixtures without building or running the solution:
+
+```bash
+python3 scripts/ci/test-assert-not-all-skipped.py
+```

scripts/ci/assert-not-all-skipped.sh

@@ -33,12 +33,31 @@ for f in "$@"; do
     exit 2
   fi
   seen=$((seen + 1))
-  # The trx <Counters .../> element in <ResultSummary> carries executed + notExecuted counts.
+  # Keep the executed-counter floor unchanged; notExecuted is only a fallback
+  # for malformed XML that this guard historically accepted.
   counters="$(grep -oE '<Counters[^>]*/?>' "$f" | head -1)"
   ex="$(printf '%s' "$counters" | grep -oE 'executed="[0-9]+"' | grep -oE '[0-9]+' || true)"
   ne="$(printf '%s' "$counters" | grep -oE 'notExecuted="[0-9]+"' | grep -oE '[0-9]+' || true)"
   ex="${ex:-0}"
   ne="${ne:-0}"
+  # VSTest can emit notExecuted="0" despite individual NotExecuted results.
+  # Count direct Results/UnitTestResult records in the document's namespace,
+  # excluding foreign elements, nested records and escaped diagnostic text.
+  ne="$(python3 - "$f" "$ne" <<'PY'
+import sys
+import xml.etree.ElementTree as ET
+
+try:
+    root = ET.parse(sys.argv[1]).getroot()
+except ET.ParseError:
+    # Skip reporting must not introduce a new malformed-XML gate failure.
+    print(sys.argv[2])
+else:
+    namespace = root.tag.split("}", 1)[0] + "}" if root.tag.startswith("{") else ""
+    results = root.findall(f"{namespace}Results/{namespace}UnitTestResult")
+    print(sum(result.get("outcome") == "NotExecuted" for result in results))
+PY
+)"
   total_executed=$((total_executed + ex))
   total_skipped=$((total_skipped + ne))
   echo "  $(basename "$f"): executed=${ex} skipped=${ne}"

scripts/ci/ci.runsettings

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <!-- #707: project scheduling is also bounded by dotnet test -m:1. -->
+  <RunConfiguration>
+    <MaxCpuCount>1</MaxCpuCount>
+  </RunConfiguration>
+  <xUnit>
+    <!-- Preserve each assembly's parallelization opt-out; bound enabled collections. -->
+    <MaxParallelThreads>2</MaxParallelThreads>
+    <ParallelAlgorithm>conservative</ParallelAlgorithm>
+  </xUnit>
+</RunSettings>

scripts/ci/test-assert-not-all-skipped.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Deterministic subprocess checks for the TRX executed-floor guard."""
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+
+SCRIPT = Path(__file__).with_name("assert-not-all-skipped.sh")
+NS = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"
+
+
+def trx(executed, outcomes, summary_skipped=0, namespace=NS, extra=""):
+    declaration = f' xmlns="{namespace}"' if namespace else ""
+    results = "".join(f'<UnitTestResult outcome="{outcome}"/>' for outcome in outcomes)
+    return (f'<TestRun{declaration}><Results>{results}{extra}</Results>'
+            f'<ResultSummary><Counters total="{len(outcomes)}" executed="{executed}" '
+            f'notExecuted="{summary_skipped}"/></ResultSummary></TestRun>')
+
+
+class SkipGuardTests(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory(prefix="trx-skip-fixtures-")
+        self.addCleanup(self.directory.cleanup)
+        self.root = Path(self.directory.name)
+
+    def write(self, name, content):
+        path = self.root / name
+        path.write_text(content, encoding="utf-8")
+        return path
+
+    def run_guard(self, *paths, minimum=None):
+        env = os.environ.copy()
+        env.pop("MIN_EXECUTED", None)
+        if minimum is not None:
+            env["MIN_EXECUTED"] = str(minimum)
+        return subprocess.run(["bash", str(SCRIPT), *map(str, paths)], env=env,
+                              text=True, capture_output=True, check=False)
+
+    def assert_report(self, result, status, executed, skipped, files=1):
+        self.assertEqual(status, result.returncode, result.stdout + result.stderr)
+        self.assertIn(f"files={files} executed={executed} skipped={skipped} ", result.stdout)
+
+    def test_mixed_results_ignore_zero_summary_skip_counter(self):
+        # A summary-only skip count misses the three actual NotExecuted records.
+        path = self.write("mixed.trx", trx(827, ["Passed"] * 827 + ["NotExecuted"] * 3))
+        result = self.run_guard(path)
+        self.assert_report(result, 0, 827, 3)
+        self.assertIn("mixed.trx: executed=827 skipped=3", result.stdout)
+
+    def test_all_skipped_fails_default_floor_but_report_only_passes(self):
+        path = self.write("all-skipped.trx", trx(0, ["NotExecuted"] * 188))
+        self.assert_report(self.run_guard(path), 1, 0, 188)
+        self.assert_report(self.run_guard(path, minimum=0), 0, 0, 188)
+
+    def test_zero_tests_fails_default_floor_but_report_only_passes(self):
+        path = self.write("zero.trx", trx(0, []))
+        self.assert_report(self.run_guard(path), 1, 0, 0)
+        self.assert_report(self.run_guard(path, minimum=0), 0, 0, 0)
+
+    def test_agreeing_summary_and_outcomes_are_not_double_counted(self):
+        path = self.write("agreeing.trx", trx(1, ["Passed", "NotExecuted"], 1))
+        self.assert_report(self.run_guard(path), 0, 1, 1)
+
+    def test_outcomes_are_authoritative_when_summary_overstates_skips(self):
+        path = self.write("overstated.trx", trx(1, ["Passed", "NotExecuted"], 9))
+        self.assert_report(self.run_guard(path), 0, 1, 1)
+
+    def test_namespace_path_and_exact_outcome_exclude_decoys(self):
+        extra = ('<Other><UnitTestResult outcome="NotExecuted"/></Other>'
+                 '<foreign:UnitTestResult xmlns:foreign="urn:other" outcome="NotExecuted"/>'
+                 '<Output><Message>&lt;UnitTestResult outcome="NotExecuted"/&gt;</Message></Output>')
+        content = trx(2, ["Passed", "Failed", "NotExecuted", "notexecuted"], extra=extra)
+        # Prefixes on Results/UnitTestResult must work just like the default namespace.
+        content = content.replace('<TestRun ', f'<TestRun xmlns:t="{NS}" ')
+        content = content.replace('<Results>', '<t:Results>').replace('</Results>', '</t:Results>')
+        content = content.replace('<UnitTestResult ', '<t:UnitTestResult ')
+        path = self.write("namespaced.trx", content)
+        self.assert_report(self.run_guard(path), 0, 2, 1)
+
+    def test_unnamespaced_results_remain_supported(self):
+        path = self.write("legacy.trx", trx(1, ["Passed", "NotExecuted"], namespace=""))
+        self.assert_report(self.run_guard(path), 0, 1, 1)
+
+    def test_executed_counter_controls_aggregate_floor_at_boundary(self):
+        # Counters retain authority even when result records disagree with executed.
+        mixed = self.write("floor.trx", trx(12000, ["Passed", "NotExecuted"]))
+        skipped = self.write("skipped.trx", trx(0, ["NotExecuted"]))
+        self.assert_report(self.run_guard(mixed, skipped, minimum=12000), 0, 12000, 2, 2)
+        self.assert_report(self.run_guard(mixed, skipped, minimum=12001), 1, 12000, 2, 2)
+
+    def test_missing_arguments_and_file_keep_exit_two(self):
+        self.assertEqual(2, self.run_guard().returncode)
+        self.assertEqual(2, self.run_guard(self.root / "missing.trx").returncode)
+
+    def test_missing_counters_keeps_existing_exit_one(self):
+        path = self.write("no-counters.trx", '<TestRun><Results/></TestRun>')
+        self.assertEqual(1, self.run_guard(path, minimum=0).returncode)
+
+    def test_malformed_xml_keeps_summary_fallback_and_gate_status(self):
+        path = self.write("malformed.trx", '<TestRun><Counters executed="2" notExecuted="3"/>')
+        self.assert_report(self.run_guard(path, minimum=2), 0, 2, 3)
+        self.assert_report(self.run_guard(path, minimum=3), 1, 2, 3)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayCeilingPagingTests.cs

@@ -197,8 +197,6 @@ public sealed class GatewayCeilingPagingTests : IDisposable
     [InlineData(L - 1, 5000u)]
     [InlineData(L, 5000u)]
     [InlineData(L + 1, 5000u)]
-    [InlineData(Ceiling - 17, 5000u)]
-    [InlineData(Ceiling - 16, 5000u)]
     [InlineData(Ceiling + 1, 5000u)]
     [InlineData(L - 2, 0u)]
     [InlineData(L - 1, 0u)]

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs

@@ -1,6 +1,9 @@
 using System.Diagnostics.CodeAnalysis;
 using Akka.Actor;
 using Akka.Cluster.Tools.PublishSubscribe;
+using Akka.Event;
+using Akka.TestKit;
+using Akka.TestKit.Xunit2;
 using Microsoft.EntityFrameworkCore;
 using Shouldly;
 using Xunit;
@@ -680,7 +683,7 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
 
     /// <summary>Counts contact loads and fails the first <c>failFirst</c> of them.</summary>
     private sealed class CountingDbFactory(
-        IDbContextFactory<OtOpcUaConfigDbContext> inner, int failFirst = 0)
+        IDbContextFactory<OtOpcUaConfigDbContext> inner, int failFirst = 0, Action<int>? onLoad = null)
         : IDbContextFactory<OtOpcUaConfigDbContext>
     {
         private int _calls;
@@ -689,7 +692,9 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
 
         public OtOpcUaConfigDbContext CreateDbContext()
         {
-            if (Interlocked.Increment(ref _calls) <= failFirst)
+            var call = Interlocked.Increment(ref _calls);
+            onLoad?.Invoke(call);
+            if (call <= failFirst)
                 throw new InvalidOperationException("ConfigDb unreachable at boot");
             return inner.CreateDbContext();
         }
@@ -747,31 +752,51 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
     [Fact]
     public void Bootstrap_retries_stop_once_the_gate_is_published()
     {
-        // The provider throws on PreStart (call 1) and on the first load's resolve (call 2), which
-        // schedules a 1 s retry. A manual refresh then publishes while that retry is still PENDING;
-        // stop-on-publish must cancel it. Without the cancel the timer fires, loads again and
-        // republishes (loads == 3, publications == 2).
+        // Freeze time only for this fact: the cluster harness and every other retry test retain
+        // their real scheduler. gRPC mode needs neither cluster membership nor pub-sub here.
+        using var local = new TestKit(@"akka {
+            loglevel = INFO
+            scheduler.implementation = ""Akka.TestKit.TestScheduler, Akka.TestKit""
+        }");
+        var scheduler = (TestScheduler)local.Sys.Scheduler;
+        var loads = local.CreateTestProbe();
+        var barrier = local.CreateTestProbe();
+        local.Sys.EventStream.Subscribe(barrier.Ref, typeof(UnhandledMessage));
         var inner = NewInMemoryDbFactory();
         SeedGrpcCluster(inner, "C1", ("host-a:4053", "host-a", 4056, true, false));
-        var factory = new CountingDbFactory(inner);
+        var factory = new CountingDbFactory(inner, onLoad: n => loads.Ref.Tell(n));
         var psk = new RecordingPskProvider();
         var calls = 0;
 
+        // PreStart and the first contact load fail resolution. The latter registers a retry.
         IMeshPskProvider? PskFactory() =>
             Interlocked.Increment(ref calls) <= 2 ? throw new InvalidOperationException("race") : psk;
 
-        var actor = Sys.ActorOf(CentralCommunicationActor.Props(
+        var actor = local.Sys.ActorOf(CentralCommunicationActor.Props(
             factory, GrpcMode(), () => null, () => new RecordingDialer(), PskFactory,
             bootstrapRetryInterval: TimeSpan.FromSeconds(1)));
+        // A mailbox barrier also waits for PreStart to register the zero-delay periodic tick.
+        var marker = new object();
+        actor.Tell(marker);
+        barrier.ExpectMsg<UnhandledMessage>(TimeSpan.FromSeconds(5)).Message.ShouldBeSameAs(marker);
+        scheduler.Advance(TimeSpan.Zero);
+        loads.ExpectMsg(1, TimeSpan.FromSeconds(5));
+        local.AwaitCondition(() => Volatile.Read(ref calls) == 2, TimeSpan.FromSeconds(5));
 
-        // Positive control: the failed first resolve happened, so a retry is pending.
-        AwaitCondition(() => Volatile.Read(ref calls) >= 2, TimeSpan.FromSeconds(5));
-        Thread.Sleep(100);
-        actor.Tell(new CentralCommunicationActor.RefreshContacts());
-        AwaitCondition(() => psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(5));
+        // This mailbox marker is handled after the failed ContactsLoaded handler returns.
+        // Unlike observing the provider call alone, it proves retry registration is complete.
+        actor.Tell(marker);
+        barrier.ExpectMsg<UnhandledMessage>(TimeSpan.FromSeconds(5)).Message.ShouldBeSameAs(marker);
 
-        Thread.Sleep(2500);
+        local.EventFilter.Info(contains: "Known-cluster gate published").ExpectOne(
+            TimeSpan.FromSeconds(5), () => actor.Tell(new CentralCommunicationActor.RefreshContacts()));
+        loads.ExpectMsg(2, TimeSpan.FromSeconds(5));
 
+        // Cross the pending retry deadline, leaving the 60-second periodic tick untouched.
+        // LoadContactsFromDb uses Task.Run, so observe its boundary for the original absence
+        // budget rather than asserting before a mistakenly retained retry can start its read.
+        scheduler.Advance(TimeSpan.FromMilliseconds(2500));
+        loads.ExpectNoMsg(TimeSpan.FromMilliseconds(2500));
         factory.Calls.ShouldBe(2);
         psk.KnownClusterPublications.Count.ShouldBe(1);
     }