merge(r2): r2-plan03

2026-07-13 11:08:27 -04:00
parent 774e3918db 3b9f6b9572
commit 3ad5ad0d7e
15 changed files with 446 additions and 13 deletions
@@ -1,3 +1,4 @@
+using Grpc.Core;
 using Microsoft.Extensions.Logging.Abstractions;
 using NSubstitute;
 using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Adapters;
@@ -47,6 +48,73 @@ public class MxGatewayDataConnectionReconnectTests
        Assert.False(loopTokens[1].IsCancellationRequested);  // new loop live
    }

+    [Fact]
+    public async Task StaleEventLoopRpcFault_AfterReconnect_DoesNotSignalDisconnected()
+    {
+        var loops = new List<TaskCompletionSource>();
+        var loopTokens = new List<CancellationToken>();
+        var adapter = CreateAdapterWithControllableLoops(loops, loopTokens, out var disconnectsRef);
+
+        var details = new Dictionary<string, string> { ["Endpoint"] = "http://gw:5000", ["ApiKey"] = "k" };
+
+        await adapter.ConnectAsync(details);
+        await WaitUntilAsync(() => loopTokens.Count == 1);
+        await adapter.ConnectAsync(details);          // reconnect: cancels loop #0, resets _disconnectFired
+        await WaitUntilAsync(() => loopTokens.Count == 2);
+
+        // The OLD loop observes its cancellation as a gRPC fault, not an OCE —
+        // the Grpc.Net default without ThrowOperationCanceledOnCancellation.
+        loops[0].SetException(new RpcException(new Status(StatusCode.Cancelled, "call cancelled")));
+        await Task.Delay(200);
+        Assert.Equal(0, Volatile.Read(ref disconnectsRef.Value));  // stale fault absorbed — the fresh connection must not flap
+
+        // The CURRENT loop's genuine fault must still signal, exactly once.
+        loops[1].SetException(new RpcException(new Status(StatusCode.Unavailable, "gateway gone")));
+        await WaitUntilAsync(() => Volatile.Read(ref disconnectsRef.Value) == 1);
+    }
+
+    [Fact]
+    public async Task StaleEventLoopObjectDisposedFault_AfterReconnect_DoesNotSignalDisconnected()
+    {
+        var loops = new List<TaskCompletionSource>();
+        var loopTokens = new List<CancellationToken>();
+        var adapter = CreateAdapterWithControllableLoops(loops, loopTokens, out var disconnectsRef);
+
+        var details = new Dictionary<string, string> { ["Endpoint"] = "http://gw:5000", ["ApiKey"] = "k" };
+
+        await adapter.ConnectAsync(details);
+        await WaitUntilAsync(() => loopTokens.Count == 1);
+        await adapter.ConnectAsync(details);          // reconnect
+        await WaitUntilAsync(() => loopTokens.Count == 2);
+
+        // The old loop faults with the concurrent-DisposeAsync shape.
+        loops[0].SetException(new ObjectDisposedException("MxGatewayClient"));
+        await Task.Delay(200);
+        Assert.Equal(0, Volatile.Read(ref disconnectsRef.Value));
+    }
+
+    private sealed class IntRef { public int Value; }
+
+    private static MxGatewayDataConnection CreateAdapterWithControllableLoops(
+        List<TaskCompletionSource> loops, List<CancellationToken> loopTokens, out IntRef disconnectsRef)
+    {
+        var factory = Substitute.For<IMxGatewayClientFactory>();
+        factory.Create().Returns(_ =>
+        {
+            var c = Substitute.For<IMxGatewayClient>();
+            var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+            loops.Add(tcs);
+            c.RunEventLoopAsync(Arg.Any<Action<MxValueUpdate>>(), Arg.Do<CancellationToken>(t => loopTokens.Add(t)))
+                .Returns(tcs.Task);
+            return c;
+        });
+        var adapter = new MxGatewayDataConnection(factory, NullLogger<MxGatewayDataConnection>.Instance);
+        var refHolder = new IntRef();
+        adapter.Disconnected += () => Interlocked.Increment(ref refHolder.Value);
+        disconnectsRef = refHolder;
+        return adapter;
+    }
+
    private static async Task WaitUntilAsync(Func<bool> condition)
    {
        for (var i = 0; i < 100 && !condition(); i++)
@@ -1054,4 +1054,31 @@ public class AlarmActorTests : TestKit, IDisposable
        var change = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
        Assert.Equal(AlarmState.Active, change.State);
    }
+
+    [Fact]
+    public void ExpressionEvalTaskFault_DoesNotKillTheAlarmTrigger()
+    {
+        var expr = CompileRawTriggerExpression("true");
+        var alarmConfig = new ResolvedAlarm
+        {
+            CanonicalName = "ExprFaultAlarm",
+            TriggerType = "Expression",
+            TriggerConfiguration = "{\"expression\":\"true\"}",
+            PriorityLevel = 1
+        };
+        var instanceProbe = CreateTestProbe();
+        var alarm = ActorOf(Props.Create(() => new AlarmActor(
+            "ExprFaultAlarm", "Pump1", instanceProbe.Ref, alarmConfig,
+            null, _sharedLibrary, _options,
+            NullLogger<AlarmActor>.Instance, expr)));
+
+        // Simulate the faulted scheduler task: must be handled (logged, in-flight
+        // cleared, false applied) — NOT an unhandled Status.Failure.
+        alarm.Tell(new AlarmActor.ExpressionEvalFailed(new ObjectDisposedException("scheduler")));
+
+        // The trigger must still be alive: the next change evaluates and activates.
+        alarm.Tell(new AttributeValueChanged("Pump1", "A", "A", 1, "Good", DateTimeOffset.UtcNow));
+        var change = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
+        Assert.Equal(AlarmState.Active, change.State);
+    }
 }
@@ -343,13 +343,41 @@ public class ScriptActorTests : TestKit, IDisposable
    private static Script<object?> CompileRawTriggerExpression(string expression)
    {
        var opts = ScriptOptions.Default
-            .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly)
+            // The test assembly is referenced so a raw-compiled expression may reference
+            // test-only hooks like EvalGate (used by the faulted-task N2 test).
+            .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly, typeof(ScriptActorTests).Assembly)
            .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
        var s = CSharpScript.Create<object?>(expression, opts, typeof(TriggerExpressionGlobals));
        s.Compile();
        return s;
    }

+    [Fact]
+    public void ExpressionEvalTaskFault_ClearsInFlight_AndDrainsPendingEvaluation()
+    {
+        var expr = CompileRawTriggerExpression(
+            "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.EvalGate.Block()");
+        var (actor, _) = CreateTriggeredActor(
+            "ExprFault", "Expression", "{\"expression\":\"true\",\"mode\":\"OnTrue\"}", null, expr);
+        try
+        {
+            actor.Tell(Change("A", "1"));                              // eval starts on the scheduler and BLOCKS → _evalInFlight = true
+            AwaitAssert(() => Assert.Equal(1, EvalGate.Entries), TimeSpan.FromSeconds(10));
+            actor.Tell(Change("A", "2"));                              // coalesces → _evalPending = true
+
+            // Simulate the faulted scheduler task the PipeTo failure mapping now surfaces
+            // (e.g. ObjectDisposedException from a disposed ScriptExecutionScheduler).
+            actor.Tell(new ScriptActor.ExpressionEvalFailed(new ObjectDisposedException("scheduler")));
+
+            // In-flight cleared + pending drained ⇒ a SECOND evaluation starts and blocks too.
+            AwaitAssert(() => Assert.Equal(2, EvalGate.Entries), TimeSpan.FromSeconds(10));
+        }
+        finally
+        {
+            EvalGate.Gate.Release(EvalGate.Entries); // ALWAYS free the shared scheduler threads
+        }
+    }
+
    [Fact]
    public void ExpressionTrigger_EvaluatesOnScriptSchedulerThread_AndStillFires()
    {
@@ -504,3 +532,17 @@ public class ScriptActorTests : TestKit, IDisposable
        instance.ExpectMsg<SetStaticAttributeCommand>(TimeSpan.FromSeconds(2));
    }
 }
+
+/// <summary>
+/// Test-only hook referenced by a raw-compiled trigger expression: <see cref="Block"/>
+/// blocks the calling script-scheduler thread on <see cref="Gate"/> so a test can hold an
+/// evaluation "in flight" and count how many evaluations have entered. Used by
+/// <c>ExpressionEvalTaskFault_ClearsInFlight_AndDrainsPendingEvaluation</c>.
+/// </summary>
+public static class EvalGate
+{
+    public static readonly SemaphoreSlim Gate = new(0);
+    private static int _entries;
+    public static int Entries => Volatile.Read(ref _entries);
+    public static bool Block() { Interlocked.Increment(ref _entries); Gate.Wait(); return false; }
+}
@@ -16,6 +16,7 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Scripts;
 /// <c>Stopwatch</c> stays allowed. The real execution-path compile against
 /// <c>ScriptGlobals</c> / <c>TriggerExpressionGlobals</c> is unchanged.
 /// </summary>
+[Collection("SiteScriptCompileCache")]
 public class ScriptCompilationServiceTests
 {
    private readonly ScriptCompilationService _service;
@@ -34,6 +35,28 @@ public class ScriptCompilationServiceTests
        Assert.Empty(result.Errors);
    }

+    [Fact]
+    public void Compile_SameCodeTwice_SharesOneRoslynCompile()
+    {
+        SiteScriptCompileCache.Clear();
+        var r1 = _service.Compile("deploy-gate-copy", "return 1 + 1;");
+        var r2 = _service.Compile("prestart-copy", "return 1 + 1;");
+
+        Assert.True(r1.IsSuccess);
+        Assert.Same(r1.CompiledScript, r2.CompiledScript);   // one compile, shared Script<T> (N4)
+        Assert.Equal(1, SiteScriptCompileCache.Hits);
+    }
+
+    [Fact]
+    public void Compile_ScriptAndTriggerExpression_DoNotCrossContaminate()
+    {
+        SiteScriptCompileCache.Clear();
+        var script = _service.Compile("s", "1 > 0");
+        var trigger = _service.CompileTriggerExpression("t", "1 > 0");
+
+        Assert.NotSame(script.CompiledScript, trigger.CompiledScript); // different globals surfaces
+    }
+
    [Fact]
    public void Compile_InvalidSyntax_ReturnsErrors()
    {
@@ -0,0 +1,54 @@
+using Microsoft.CodeAnalysis.CSharp.Scripting;
+using Microsoft.CodeAnalysis.Scripting;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Scripts;
+
+/// <summary>
+/// Tests for the process-wide <see cref="SiteScriptCompileCache"/>. The cache is
+/// static/process-wide state, so this class shares the "SiteScriptCompileCache" xunit
+/// collection with <c>ScriptCompilationServiceTests</c> (no cross-class parallelism) and
+/// calls <see cref="SiteScriptCompileCache.Clear"/> at the top of each test.
+/// </summary>
+[Collection("SiteScriptCompileCache")]
+public class SiteScriptCompileCacheTests
+{
+    private static ScriptCompilationResult Ok() =>
+        ScriptCompilationResult.Succeeded(CSharpScript.Create<object?>("1", ScriptOptions.Default, typeof(ScriptGlobals)));
+
+    [Fact]
+    public void GetOrAdd_SameCodeAndGlobals_ComputesOnce()
+    {
+        SiteScriptCompileCache.Clear();
+        var factoryCalls = 0;
+        ScriptCompilationResult Factory() { factoryCalls++; return Ok(); }
+
+        var r1 = SiteScriptCompileCache.GetOrAdd("return 1;", typeof(ScriptGlobals), Factory);
+        var r2 = SiteScriptCompileCache.GetOrAdd("return 1;", typeof(ScriptGlobals), Factory);
+
+        Assert.Equal(1, factoryCalls);
+        Assert.Same(r1, r2);
+        Assert.Equal(1, SiteScriptCompileCache.Hits);
+    }
+
+    [Fact]
+    public void GetOrAdd_SameCode_DifferentGlobals_AreSeparateEntries()
+    {
+        SiteScriptCompileCache.Clear();
+        SiteScriptCompileCache.GetOrAdd("1 > 0", typeof(ScriptGlobals), Ok);
+        SiteScriptCompileCache.GetOrAdd("1 > 0", typeof(TriggerExpressionGlobals), Ok);
+
+        Assert.Equal(2, SiteScriptCompileCache.Count);
+        Assert.Equal(0, SiteScriptCompileCache.Hits); // no cross-globals hit
+    }
+
+    [Fact]
+    public void Overflow_ClearsWholesale()
+    {
+        SiteScriptCompileCache.Clear();
+        for (var i = 0; i <= SiteScriptCompileCache.MaxEntries; i++)
+            SiteScriptCompileCache.GetOrAdd($"return {i};", typeof(ScriptGlobals), Ok);
+
+        Assert.True(SiteScriptCompileCache.Count <= SiteScriptCompileCache.MaxEntries);
+    }
+}
@@ -40,6 +40,24 @@ public class SiteRuntimeOptionsValidatorTests
        Assert.Contains("ScriptExecutionThreadCount", result.FailureMessage);
    }

+    [Fact]
+    public void ZeroTagSubscribeRetryIntervalMs_IsRejected()
+    {
+        var result = Validate(new SiteRuntimeOptions { TagSubscribeRetryIntervalMs = 0 });
+
+        Assert.True(result.Failed);
+        Assert.Contains("TagSubscribeRetryIntervalMs", result.FailureMessage);
+    }
+
+    [Fact]
+    public void NegativeStuckScriptGraceMs_IsRejected()
+    {
+        var result = Validate(new SiteRuntimeOptions { StuckScriptGraceMs = -1 });
+
+        Assert.True(result.Failed);
+        Assert.Contains("StuckScriptGraceMs", result.FailureMessage);
+    }
+
    [Fact]
    public void ZeroStreamBufferSize_IsRejected()
    {