using Akka.Actor; using Akka.TestKit; using Akka.TestKit.Xunit2; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; using ZB.MOM.WW.ScadaBridge.TestSupport; using System.Reflection; using System.Text.Json; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors; /// /// Regression coverage: the Instance Actor must not hand its /// own live mutable _attributes dictionary by reference into the /// / constructors. /// /// Each child constructor runs on the child's own mailbox thread and seeds itself /// by enumerating the dictionary it was given. The Instance Actor concurrently /// mutates _attributes in HandleAttributeValueChanged / /// HandleTagValueUpdate. is not safe /// for concurrent read/write: if a child enumerates the shared live dictionary /// while the Instance Actor inserts into it, the child constructor throws /// ("collection was modified") — surfacing /// as ActorInitializationException and stopping the child. /// /// The fix: CreateChildActors snapshots _attributes once on the /// Instance Actor thread (new Dictionary<,>(_attributes)) and hands /// each child that private copy. This test asserts the isolation contract /// directly and deterministically: every child's seed dictionary must be a /// distinct object from the Instance Actor's live _attributes, while still /// carrying the same point-in-time contents. /// public class InstanceActorChildAttributeRaceTests : TestKit, IDisposable { private readonly SiteStorageService _storage; private readonly ScriptCompilationService _compilationService; private readonly SharedScriptLibrary _sharedScriptLibrary; private readonly SiteRuntimeOptions _options; private readonly TestLocalDb _localDb; /// Initializes the temporary database, storage, compilation service and options used by the tests. public InstanceActorChildAttributeRaceTests() { _localDb = TestLocalDb.CreateTemp("instance-race-test"); _storage = new SiteStorageService( _localDb.Db, NullLogger.Instance); _storage.InitializeAsync().GetAwaiter().GetResult(); _compilationService = new ScriptCompilationService( NullLogger.Instance); _sharedScriptLibrary = new SharedScriptLibrary( _compilationService, NullLogger.Instance); _options = new SiteRuntimeOptions { MaxScriptCallDepth = 10, ScriptExecutionTimeoutSeconds = 30 }; } /// void IDisposable.Dispose() { // TestKit teardown first, so no in-flight actor can reach a disposed ILocalDb; // then dispose the database before deleting — the master connection anchors the WAL. Shutdown(); var path = _localDb.Path; _localDb.Dispose(); TestLocalDb.DeleteFiles(path); } private static FlattenedConfiguration BuildConfig(string instanceName) => new() { InstanceUniqueName = instanceName, Attributes = [ new ResolvedAttribute { CanonicalName = "Temperature", Value = "98.6", DataType = "Double" }, new ResolvedAttribute { CanonicalName = "Pressure", Value = "12", DataType = "Int32" }, new ResolvedAttribute { CanonicalName = "Label", Value = "Main Pump", DataType = "String" } ], Scripts = [ new ResolvedScript { CanonicalName = "WorkerA", Code = "return 1;", TriggerType = "ValueChange", TriggerConfiguration = "{\"AttributeName\":\"Temperature\"}" }, new ResolvedScript { CanonicalName = "WorkerB", Code = "return 2;", TriggerType = "ValueChange", TriggerConfiguration = "{\"AttributeName\":\"Pressure\"}" } ], Alarms = [ new ResolvedAlarm { CanonicalName = "HighTemp", TriggerType = "ValueMatch", TriggerConfiguration = "{}", PriorityLevel = 1 } ] }; /// /// Resolves a named child and the live actor instance behind it, retrying until BOTH /// exist (register row 40 — absence/settle timing pins). /// /// Two distinct publication races sit between "the parent reported N children" and /// "the child's actor object is reachable by reflection", and neither is ordered against /// the test thread: throws /// until the child's cell is published on the parent, /// and the cell's internal Actor property stays null until the child's constructor /// has run on the child's own mailbox thread. A single-shot resolve therefore fails /// intermittently under load — observed both ways in full-suite runs ("subject was null" /// from ResolveOne, and a null instance from the reflection read). Retrying to a bounded /// deadline removes the race without weakening the assertion: what the test proves is /// still the seed-dictionary isolation contract, checked once both objects exist. /// private async Task ResolveChildInstanceAsync(IActorRef parent, string childName) where T : class { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15); while (true) { try { var child = await Sys.ActorSelection(parent.Path / childName) .ResolveOne(TimeSpan.FromSeconds(1)); var cell = ((ActorRefWithCell)child).Underlying; // ActorCell exposes the actor instance via its internal Actor property. var actorProp = cell.GetType().GetProperty( "Actor", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); if (actorProp!.GetValue(cell) is T typed) { return typed; } } catch (ActorNotFoundException) { // The child's cell is not published on the parent yet — retry below. } if (DateTime.UtcNow > deadline) { throw new TimeoutException( $"Child actor '{childName}' (or its {typeof(T).Name} instance) never " + "materialised within the resolve deadline."); } await Task.Delay(25); } } private static Dictionary GetPrivateAttributes(InstanceActor instance) { var field = typeof(InstanceActor).GetField( "_attributes", BindingFlags.Instance | BindingFlags.NonPublic); return (Dictionary)field!.GetValue(instance)!; } /// Verifies that child actors are seeded from an isolated copy of the attributes, not the live attributes dictionary. /// A task that represents the asynchronous operation. [Fact] public async Task ChildActors_AreSeededFromAnIsolatedCopy_NotTheLiveAttributesDictionary() { const string instanceName = "RacePump"; var config = BuildConfig(instanceName); var testRef = ActorOfAsTestActorRef( Props.Create(() => new InstanceActor( instanceName, JsonSerializer.Serialize(config), _storage, _compilationService, _sharedScriptLibrary, null, _options, NullLogger.Instance)), "instance"); var instanceActor = testRef.UnderlyingActor; var liveAttributes = GetPrivateAttributes(instanceActor); // Sanity: the children were created. Assert.Equal(2, instanceActor.ScriptActorCount); Assert.Equal(1, instanceActor.AlarmActorCount); // Every child Script Actor must have been seeded from a dictionary that // is NOT the Instance Actor's live _attributes field — otherwise the // child constructor would enumerate a dictionary the Instance Actor // mutates on another thread. foreach (var name in new[] { "WorkerA", "WorkerB" }) { var scriptActor = await ResolveChildInstanceAsync(testRef, $"script-{name}"); Assert.NotNull(scriptActor.SeedAttributesReference); Assert.False( ReferenceEquals(scriptActor.SeedAttributesReference, liveAttributes), $"Script Actor '{name}' was seeded from the Instance Actor's live " + "_attributes dictionary by reference (SiteRuntime-017). It must be " + "given a private snapshot copy."); // The snapshot must still carry the same point-in-time contents. Assert.Equal(liveAttributes.Count, scriptActor.SeedAttributesReference!.Count); foreach (var kvp in liveAttributes) { Assert.True(scriptActor.SeedAttributesReference.ContainsKey(kvp.Key)); } } // The Alarm Actor must likewise be seeded from an isolated copy. var alarmActor = await ResolveChildInstanceAsync(testRef, "alarm-HighTemp"); Assert.NotNull(alarmActor.SeedAttributesReference); Assert.False( ReferenceEquals(alarmActor.SeedAttributesReference, liveAttributes), "Alarm Actor 'HighTemp' was seeded from the Instance Actor's live " + "_attributes dictionary by reference (SiteRuntime-017). It must be " + "given a private snapshot copy."); Assert.Equal(liveAttributes.Count, alarmActor.SeedAttributesReference!.Count); } }