using Akka.Actor;
using Akka.TestKit;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.TestSupport;
using System.Reflection;
using System.Text.Json;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
///
/// Regression coverage: the Instance Actor must not hand its
/// own live mutable _attributes dictionary by reference into the
/// / constructors.
///
/// Each child constructor runs on the child's own mailbox thread and seeds itself
/// by enumerating the dictionary it was given. The Instance Actor concurrently
/// mutates _attributes in HandleAttributeValueChanged /
/// HandleTagValueUpdate. is not safe
/// for concurrent read/write: if a child enumerates the shared live dictionary
/// while the Instance Actor inserts into it, the child constructor throws
/// ("collection was modified") — surfacing
/// as ActorInitializationException and stopping the child.
///
/// The fix: CreateChildActors snapshots _attributes once on the
/// Instance Actor thread (new Dictionary<,>(_attributes)) and hands
/// each child that private copy. This test asserts the isolation contract
/// directly and deterministically: every child's seed dictionary must be a
/// distinct object from the Instance Actor's live _attributes, while still
/// carrying the same point-in-time contents.
///
public class InstanceActorChildAttributeRaceTests : TestKit, IDisposable
{
private readonly SiteStorageService _storage;
private readonly ScriptCompilationService _compilationService;
private readonly SharedScriptLibrary _sharedScriptLibrary;
private readonly SiteRuntimeOptions _options;
private readonly TestLocalDb _localDb;
/// Initializes the temporary database, storage, compilation service and options used by the tests.
public InstanceActorChildAttributeRaceTests()
{
_localDb = TestLocalDb.CreateTemp("instance-race-test");
_storage = new SiteStorageService(
_localDb.Db,
NullLogger.Instance);
_storage.InitializeAsync().GetAwaiter().GetResult();
_compilationService = new ScriptCompilationService(
NullLogger.Instance);
_sharedScriptLibrary = new SharedScriptLibrary(
_compilationService, NullLogger.Instance);
_options = new SiteRuntimeOptions
{
MaxScriptCallDepth = 10,
ScriptExecutionTimeoutSeconds = 30
};
}
///
void IDisposable.Dispose()
{
// TestKit teardown first, so no in-flight actor can reach a disposed ILocalDb;
// then dispose the database before deleting — the master connection anchors the WAL.
Shutdown();
var path = _localDb.Path;
_localDb.Dispose();
TestLocalDb.DeleteFiles(path);
}
private static FlattenedConfiguration BuildConfig(string instanceName)
=> new()
{
InstanceUniqueName = instanceName,
Attributes =
[
new ResolvedAttribute { CanonicalName = "Temperature", Value = "98.6", DataType = "Double" },
new ResolvedAttribute { CanonicalName = "Pressure", Value = "12", DataType = "Int32" },
new ResolvedAttribute { CanonicalName = "Label", Value = "Main Pump", DataType = "String" }
],
Scripts =
[
new ResolvedScript
{
CanonicalName = "WorkerA", Code = "return 1;",
TriggerType = "ValueChange",
TriggerConfiguration = "{\"AttributeName\":\"Temperature\"}"
},
new ResolvedScript
{
CanonicalName = "WorkerB", Code = "return 2;",
TriggerType = "ValueChange",
TriggerConfiguration = "{\"AttributeName\":\"Pressure\"}"
}
],
Alarms =
[
new ResolvedAlarm
{
CanonicalName = "HighTemp",
TriggerType = "ValueMatch",
TriggerConfiguration = "{}",
PriorityLevel = 1
}
]
};
///
/// Resolves a named child and the live actor instance behind it, retrying until BOTH
/// exist (register row 40 — absence/settle timing pins).
///
/// Two distinct publication races sit between "the parent reported N children" and
/// "the child's actor object is reachable by reflection", and neither is ordered against
/// the test thread: throws
/// until the child's cell is published on the parent,
/// and the cell's internal Actor property stays null until the child's constructor
/// has run on the child's own mailbox thread. A single-shot resolve therefore fails
/// intermittently under load — observed both ways in full-suite runs ("subject was null"
/// from ResolveOne, and a null instance from the reflection read). Retrying to a bounded
/// deadline removes the race without weakening the assertion: what the test proves is
/// still the seed-dictionary isolation contract, checked once both objects exist.
///
private async Task ResolveChildInstanceAsync(IActorRef parent, string childName)
where T : class
{
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15);
while (true)
{
try
{
var child = await Sys.ActorSelection(parent.Path / childName)
.ResolveOne(TimeSpan.FromSeconds(1));
var cell = ((ActorRefWithCell)child).Underlying;
// ActorCell exposes the actor instance via its internal Actor property.
var actorProp = cell.GetType().GetProperty(
"Actor", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
if (actorProp!.GetValue(cell) is T typed)
{
return typed;
}
}
catch (ActorNotFoundException)
{
// The child's cell is not published on the parent yet — retry below.
}
if (DateTime.UtcNow > deadline)
{
throw new TimeoutException(
$"Child actor '{childName}' (or its {typeof(T).Name} instance) never " +
"materialised within the resolve deadline.");
}
await Task.Delay(25);
}
}
private static Dictionary GetPrivateAttributes(InstanceActor instance)
{
var field = typeof(InstanceActor).GetField(
"_attributes", BindingFlags.Instance | BindingFlags.NonPublic);
return (Dictionary)field!.GetValue(instance)!;
}
/// Verifies that child actors are seeded from an isolated copy of the attributes, not the live attributes dictionary.
/// A task that represents the asynchronous operation.
[Fact]
public async Task ChildActors_AreSeededFromAnIsolatedCopy_NotTheLiveAttributesDictionary()
{
const string instanceName = "RacePump";
var config = BuildConfig(instanceName);
var testRef = ActorOfAsTestActorRef(
Props.Create(() => new InstanceActor(
instanceName,
JsonSerializer.Serialize(config),
_storage,
_compilationService,
_sharedScriptLibrary,
null,
_options,
NullLogger.Instance)),
"instance");
var instanceActor = testRef.UnderlyingActor;
var liveAttributes = GetPrivateAttributes(instanceActor);
// Sanity: the children were created.
Assert.Equal(2, instanceActor.ScriptActorCount);
Assert.Equal(1, instanceActor.AlarmActorCount);
// Every child Script Actor must have been seeded from a dictionary that
// is NOT the Instance Actor's live _attributes field — otherwise the
// child constructor would enumerate a dictionary the Instance Actor
// mutates on another thread.
foreach (var name in new[] { "WorkerA", "WorkerB" })
{
var scriptActor = await ResolveChildInstanceAsync(testRef, $"script-{name}");
Assert.NotNull(scriptActor.SeedAttributesReference);
Assert.False(
ReferenceEquals(scriptActor.SeedAttributesReference, liveAttributes),
$"Script Actor '{name}' was seeded from the Instance Actor's live " +
"_attributes dictionary by reference (SiteRuntime-017). It must be " +
"given a private snapshot copy.");
// The snapshot must still carry the same point-in-time contents.
Assert.Equal(liveAttributes.Count, scriptActor.SeedAttributesReference!.Count);
foreach (var kvp in liveAttributes)
{
Assert.True(scriptActor.SeedAttributesReference.ContainsKey(kvp.Key));
}
}
// The Alarm Actor must likewise be seeded from an isolated copy.
var alarmActor = await ResolveChildInstanceAsync(testRef, "alarm-HighTemp");
Assert.NotNull(alarmActor.SeedAttributesReference);
Assert.False(
ReferenceEquals(alarmActor.SeedAttributesReference, liveAttributes),
"Alarm Actor 'HighTemp' was seeded from the Instance Actor's live " +
"_attributes dictionary by reference (SiteRuntime-017). It must be " +
"given a private snapshot copy.");
Assert.Equal(liveAttributes.Count, alarmActor.SeedAttributesReference!.Count);
}
}