ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/InstanceActorChildAttributeRaceTests.cs

using Akka.Actor;
using Akka.TestKit;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.TestSupport;
using System.Reflection;
using System.Text.Json;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// Regression coverage: the Instance Actor must not hand its
/// own live mutable <c>_attributes</c> dictionary by reference into the
/// <see cref="ScriptActor"/> / <see cref="AlarmActor"/> constructors.
///
/// Each child constructor runs on the child's own mailbox thread and seeds itself
/// by enumerating the dictionary it was given. The Instance Actor concurrently
/// mutates <c>_attributes</c> in <c>HandleAttributeValueChanged</c> /
/// <c>HandleTagValueUpdate</c>. <see cref="Dictionary{TKey,TValue}"/> is not safe
/// for concurrent read/write: if a child enumerates the shared live dictionary
/// while the Instance Actor inserts into it, the child constructor throws
/// <see cref="InvalidOperationException"/> ("collection was modified") — surfacing
/// as <c>ActorInitializationException</c> and stopping the child.
///
/// The fix: <c>CreateChildActors</c> snapshots <c>_attributes</c> once on the
/// Instance Actor thread (<c>new Dictionary&lt;,&gt;(_attributes)</c>) and hands
/// each child that private copy. This test asserts the isolation contract
/// directly and deterministically: every child's seed dictionary must be a
/// distinct object from the Instance Actor's live <c>_attributes</c>, while still
/// carrying the same point-in-time contents.
/// </summary>
public class InstanceActorChildAttributeRaceTests : TestKit, IDisposable
{
    private readonly SiteStorageService _storage;
    private readonly ScriptCompilationService _compilationService;
    private readonly SharedScriptLibrary _sharedScriptLibrary;
    private readonly SiteRuntimeOptions _options;
    private readonly TestLocalDb _localDb;

    /// <summary>Initializes the temporary database, storage, compilation service and options used by the tests.</summary>
    public InstanceActorChildAttributeRaceTests()
    {
        _localDb = TestLocalDb.CreateTemp("instance-race-test");
        _storage = new SiteStorageService(
            _localDb.Db,
            NullLogger<SiteStorageService>.Instance);
        _storage.InitializeAsync().GetAwaiter().GetResult();
        _compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedScriptLibrary = new SharedScriptLibrary(
            _compilationService, NullLogger<SharedScriptLibrary>.Instance);
        _options = new SiteRuntimeOptions
        {
            MaxScriptCallDepth = 10,
            ScriptExecutionTimeoutSeconds = 30
        };
    }

    /// <inheritdoc />
    void IDisposable.Dispose()
    {
        // TestKit teardown first, so no in-flight actor can reach a disposed ILocalDb;
        // then dispose the database before deleting — the master connection anchors the WAL.
        Shutdown();
        var path = _localDb.Path;
        _localDb.Dispose();
        TestLocalDb.DeleteFiles(path);
    }

    private static FlattenedConfiguration BuildConfig(string instanceName)
        => new()
        {
            InstanceUniqueName = instanceName,
            Attributes =
            [
                new ResolvedAttribute { CanonicalName = "Temperature", Value = "98.6", DataType = "Double" },
                new ResolvedAttribute { CanonicalName = "Pressure", Value = "12", DataType = "Int32" },
                new ResolvedAttribute { CanonicalName = "Label", Value = "Main Pump", DataType = "String" }
            ],
            Scripts =
            [
                new ResolvedScript
                {
                    CanonicalName = "WorkerA", Code = "return 1;",
                    TriggerType = "ValueChange",
                    TriggerConfiguration = "{\"AttributeName\":\"Temperature\"}"
                },
                new ResolvedScript
                {
                    CanonicalName = "WorkerB", Code = "return 2;",
                    TriggerType = "ValueChange",
                    TriggerConfiguration = "{\"AttributeName\":\"Pressure\"}"
                }
            ],
            Alarms =
            [
                new ResolvedAlarm
                {
                    CanonicalName = "HighTemp",
                    TriggerType = "ValueMatch",
                    TriggerConfiguration = "{}",
                    PriorityLevel = 1
                }
            ]
        };

    /// <summary>
    /// Resolves a named child and the live actor instance behind it, retrying until BOTH
    /// exist (register row 40 — absence/settle timing pins).
    ///
    /// <para>Two distinct publication races sit between "the parent reported N children" and
    /// "the child's actor object is reachable by reflection", and neither is ordered against
    /// the test thread: <see cref="ActorSelection.ResolveOne"/> throws
    /// <see cref="ActorNotFoundException"/> until the child's cell is published on the parent,
    /// and the cell's internal <c>Actor</c> property stays null until the child's constructor
    /// has run on the child's own mailbox thread. A single-shot resolve therefore fails
    /// intermittently under load — observed both ways in full-suite runs ("subject was null"
    /// from ResolveOne, and a null instance from the reflection read). Retrying to a bounded
    /// deadline removes the race without weakening the assertion: what the test proves is
    /// still the seed-dictionary isolation contract, checked once both objects exist.</para>
    /// </summary>
    private async Task<T> ResolveChildInstanceAsync<T>(IActorRef parent, string childName)
        where T : class
    {
        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15);
        while (true)
        {
            try
            {
                var child = await Sys.ActorSelection(parent.Path / childName)
                    .ResolveOne(TimeSpan.FromSeconds(1));
                var cell = ((ActorRefWithCell)child).Underlying;
                // ActorCell exposes the actor instance via its internal Actor property.
                var actorProp = cell.GetType().GetProperty(
                    "Actor", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
                if (actorProp!.GetValue(cell) is T typed)
                {
                    return typed;
                }
            }
            catch (ActorNotFoundException)
            {
                // The child's cell is not published on the parent yet — retry below.
            }

            if (DateTime.UtcNow > deadline)
            {
                throw new TimeoutException(
                    $"Child actor '{childName}' (or its {typeof(T).Name} instance) never " +
                    "materialised within the resolve deadline.");
            }

            await Task.Delay(25);
        }
    }

    private static Dictionary<string, object?> GetPrivateAttributes(InstanceActor instance)
    {
        var field = typeof(InstanceActor).GetField(
            "_attributes", BindingFlags.Instance | BindingFlags.NonPublic);
        return (Dictionary<string, object?>)field!.GetValue(instance)!;
    }

    /// <summary>Verifies that child actors are seeded from an isolated copy of the attributes, not the live attributes dictionary.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task ChildActors_AreSeededFromAnIsolatedCopy_NotTheLiveAttributesDictionary()
    {
        const string instanceName = "RacePump";
        var config = BuildConfig(instanceName);

        var testRef = ActorOfAsTestActorRef<InstanceActor>(
            Props.Create(() => new InstanceActor(
                instanceName,
                JsonSerializer.Serialize(config),
                _storage,
                _compilationService,
                _sharedScriptLibrary,
                null,
                _options,
                NullLogger<InstanceActor>.Instance)),
            "instance");

        var instanceActor = testRef.UnderlyingActor;
        var liveAttributes = GetPrivateAttributes(instanceActor);

        // Sanity: the children were created.
        Assert.Equal(2, instanceActor.ScriptActorCount);
        Assert.Equal(1, instanceActor.AlarmActorCount);

        // Every child Script Actor must have been seeded from a dictionary that
        // is NOT the Instance Actor's live _attributes field — otherwise the
        // child constructor would enumerate a dictionary the Instance Actor
        // mutates on another thread.
        foreach (var name in new[] { "WorkerA", "WorkerB" })
        {
            var scriptActor = await ResolveChildInstanceAsync<ScriptActor>(testRef, $"script-{name}");

            Assert.NotNull(scriptActor.SeedAttributesReference);
            Assert.False(
                ReferenceEquals(scriptActor.SeedAttributesReference, liveAttributes),
                $"Script Actor '{name}' was seeded from the Instance Actor's live " +
                "_attributes dictionary by reference (SiteRuntime-017). It must be " +
                "given a private snapshot copy.");

            // The snapshot must still carry the same point-in-time contents.
            Assert.Equal(liveAttributes.Count, scriptActor.SeedAttributesReference!.Count);
            foreach (var kvp in liveAttributes)
            {
                Assert.True(scriptActor.SeedAttributesReference.ContainsKey(kvp.Key));
            }
        }

        // The Alarm Actor must likewise be seeded from an isolated copy.
        var alarmActor = await ResolveChildInstanceAsync<AlarmActor>(testRef, "alarm-HighTemp");

        Assert.NotNull(alarmActor.SeedAttributesReference);
        Assert.False(
            ReferenceEquals(alarmActor.SeedAttributesReference, liveAttributes),
            "Alarm Actor 'HighTemp' was seeded from the Instance Actor's live " +
            "_attributes dictionary by reference (SiteRuntime-017). It must be " +
            "given a private snapshot copy.");
        Assert.Equal(liveAttributes.Count, alarmActor.SeedAttributesReference!.Count);
    }
}