lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorUnreadableArtifactTests.cs

using System.Text.Json;
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;

namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;

/// <summary>
///     Issue #485 (driver-side half) — an artifact the host could not obtain must not be mistaken for a
///     configuration that contains nothing.
/// </summary>
/// <remarks>
///     <para>
///         <c>ReconcileDrivers</c> and <c>PushDesiredSubscriptions</c> both read the artifact as
///         <c>…FirstOrDefault() ?? Array.Empty&lt;byte&gt;()</c>. The <b>throw</b> path in each already
///         returns early, but a row that is absent (or carries a zero-length blob) yields empty bytes that
///         flow onwards as a real answer: zero driver specs, so <c>DriverSpawnPlanner</c> plans every
///         running child for <c>StopChild</c>, and then an empty desired-subscription set drops each
///         surviving driver's live handle. A node loses its entire field I/O and still ACKs Applied.
///     </para>
///     <para>
///         The same reasoning as the address-space half applies: nothing legitimate produces a zero-length
///         blob — an operator who really deletes every driver still deploys a JSON document with empty
///         arrays — so "no bytes" can only mean the read did not answer. Seeding a row whose
///         <c>ArtifactBlob</c> is empty reproduces exactly the bytes the missing-row case delivers, without
///         needing to race a delete against the two reads.
///     </para>
/// </remarks>
public sealed class DriverHostActorUnreadableArtifactTests : RuntimeActorTestBase
{
    private static readonly NodeId TestNode = NodeId.Parse("driver-test");
    private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
    private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
    private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);

    /// <summary>How long to let post-ACK subscription traffic settle before asserting it did NOT happen.</summary>
    private static readonly TimeSpan SettleWindow = TimeSpan.FromSeconds(2);

    private const string SpeedRef = "Plant/Modbus/dev1/speed";

    /// <summary>A dispatch whose artifact reads back as no bytes must leave the running drivers — and their
    /// live subscriptions — exactly as they were.</summary>
    [Fact]
    public void Dispatch_whose_artifact_reads_back_empty_keeps_the_drivers_and_their_subscriptions()
    {
        var db = NewInMemoryDbFactory();
        var factory = new SubscribingDriverFactory("Modbus");
        var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);

        // Baseline: the child is up and subscribed to its one raw tag.
        AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
        AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1");

        // The next dispatch's artifact comes back as no bytes at all.
        actor.Tell(new DispatchDeployment(SeedUnreadableDeployment(db, RevB), RevB, CorrelationId.NewId()));
        coordinator.ExpectMsg<ApplyAck>(Timeout);

        // The ACK precedes the SubscribeBulk pass, and the child's unsubscribe is a further async self-tell,
        // so settle before asserting an ABSENCE. SettleWindow is calibrated by
        // <see cref="Dropping_a_drivers_last_tag_does_clear_its_subscription"/>, which observes the very same
        // unsubscribe ARRIVING well inside it — without that control this assertion would pass on a race.
        AskDiagnostics(actor);                       // ordering barrier through the host's mailbox
        Thread.Sleep(SettleWindow);

        AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1");  // not stopped
        factory.UnsubscribeCount.ShouldBe(0);                                       // handle not dropped
        factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef);
    }

    /// <summary>
    ///     Positive control for the subscription half: a READABLE artifact that drops the driver's last tag
    ///     genuinely does tear the live subscription down, and does so well within
    ///     <see cref="SettleWindow"/> — without this, the absence asserted above would pass on a race.
    ///     <para><b>The observable changed with #516.</b> Dropping a tag changes the driver's
    ///     <c>DriverConfig</c>, which used to be an in-place delta: the child received an empty desired set
    ///     and called <c>UnsubscribeAsync</c>. A config change is now a stop + respawn, so the old child is
    ///     STOPPED (taking its subscription with it via <c>ShutdownAsync</c>) and a fresh one spawns with
    ///     nothing to subscribe to. The teardown is therefore observed as a shutdown, not an unsubscribe —
    ///     the same event, reached by a different route.</para>
    /// </summary>
    [Fact]
    public void Dropping_a_drivers_last_tag_does_clear_its_subscription()
    {
        var db = NewInMemoryDbFactory();
        var factory = new SubscribingDriverFactory("Modbus");
        var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);

        AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
        factory.ShutdownCount.ShouldBe(0);

        actor.Tell(new DispatchDeployment(SeedTaglessDriverDeployment(db, RevB), RevB, CorrelationId.NewId()));
        coordinator.ExpectMsg<ApplyAck>(Timeout);

        AwaitAssert(() => factory.ShutdownCount.ShouldBeGreaterThanOrEqualTo(1), duration: SettleWindow);
        AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"); // respawned under the same id
    }

    /// <summary>Positive control: the guard keys on "the read gave us nothing", not on "the new config has
    /// fewer drivers". A READABLE artifact that genuinely drops the driver still stops it — otherwise the
    /// test above would pass just as happily against a host that had stopped reconciling altogether.</summary>
    [Fact]
    public void Dispatch_whose_readable_artifact_genuinely_drops_the_driver_still_stops_it()
    {
        var db = NewInMemoryDbFactory();
        var factory = new SubscribingDriverFactory("Modbus");
        var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);

        AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);

        // A real artifact that simply carries no drivers — an operator deleting the last driver.
        actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
        coordinator.ExpectMsg<ApplyAck>(Timeout);

        AwaitAssert(
            () => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
            duration: Timeout);
    }

    /// <summary>
    ///     Issue #486 — an apply that read no artifact applied nothing, so it must NOT be reported as
    ///     Applied. Reporting success also advanced <c>_currentRevision</c>, and because
    ///     <c>HandleDispatchFromSteady</c> short-circuits on a revision match, the node could never be
    ///     healed by re-dispatching that revision.
    /// </summary>
    [Fact]
    public void Dispatch_whose_artifact_reads_back_empty_acks_Failed_and_leaves_the_revision_alone()
    {
        var db = NewInMemoryDbFactory();
        var factory = new SubscribingDriverFactory("Modbus");
        var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
        AskDiagnostics(actor).CurrentRevision.ShouldBe(RevA);

        actor.Tell(new DispatchDeployment(SeedUnreadableDeployment(db, RevB), RevB, CorrelationId.NewId()));
        var ack = coordinator.ExpectMsg<ApplyAck>(Timeout);

        ack.Outcome.ShouldBe(ApplyAckOutcome.Failed);
        ack.FailureReason.ShouldNotBeNullOrWhiteSpace();

        var snapshot = AskDiagnostics(actor);
        snapshot.CurrentRevision.ShouldBe(RevA);                     // never applied ⇒ never claimed
        snapshot.Drivers.Select(d => d.Name).ShouldContain("drv-1"); // …and #485 still holds
    }

    /// <summary>Issue #486 — the point of failing the apply: the SAME revision can be dispatched again and
    /// now actually applies, instead of being waved through by the "already at this rev" short-circuit. The
    /// recovered artifact adds a second driver, so a short-circuited ACK (which has no side effects) cannot
    /// satisfy this test — drv-2 only appears if the artifact was genuinely re-read and reconciled.</summary>
    [Fact]
    public void A_failed_apply_is_retried_not_short_circuited_once_the_artifact_is_readable_again()
    {
        var db = NewInMemoryDbFactory();
        var factory = new SubscribingDriverFactory("Modbus");
        var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);

        var deploymentId = SeedUnreadableDeployment(db, RevB);
        actor.Tell(new DispatchDeployment(deploymentId, RevB, CorrelationId.NewId()));
        coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Failed);

        // The ConfigDb recovers: the row now carries a real artifact, under the SAME revision.
        SetArtifact(db, deploymentId, TwoDriverArtifact());
        actor.Tell(new DispatchDeployment(deploymentId, RevB, CorrelationId.NewId()));

        coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
        AwaitAssert(
            () => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-2"),
            duration: Timeout);
        AskDiagnostics(actor).CurrentRevision.ShouldBe(RevB);
    }

    /// <summary>
    ///     #609 — removing a driver instance from the plan evicts its cached telemetry snapshots, so a new
    ///     subscriber is no longer primed with a decommissioned driver's last Health/Resilience. The
    ///     surviving instance's snapshot is the positive control: eviction is per-instance, not a wipe.
    /// </summary>
    [Fact]
    public void Removing_a_driver_instance_evicts_its_telemetry_snapshots_but_not_a_neighbours()
    {
        var db = NewInMemoryDbFactory();
        var factory = new SubscribingDriverFactory("Modbus");
        var hub = new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub();
        var coordinator = CreateTestProbe();
        var actor = Sys.ActorOf(DriverHostActor.Props(
            db, TestNode, coordinator.Ref,
            driverFactory: factory,
            localRoles: new HashSet<string> { "driver" },
            telemetryHub: hub));

        actor.Tell(new DispatchDeployment(SeedV3Deployment(db, RevA), RevA, CorrelationId.NewId()));
        coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);

        hub.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health(new(
            "c1", "drv-1", "Healthy", null, null, 0, DateTime.UtcNow)));
        hub.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health(new(
            "c1", "other-node-driver", "Healthy", null, null, 0, DateTime.UtcNow)));

        // Control: before the removal, drv-1 replays.
        using (var before = hub.Subscribe(16))
        {
            var seen = new List<string>();
            while (before.Reader.TryRead(out var i))
                if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health h) seen.Add(h.E.DriverInstanceId);
            seen.ShouldContain("drv-1");
        }

        actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
        coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
        AwaitAssert(
            () => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
            duration: Timeout);

        using var after = hub.Subscribe(16);
        var replayed = new List<string>();
        while (after.Reader.TryRead(out var i))
            if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health h) replayed.Add(h.E.DriverInstanceId);
        replayed.ShouldNotContain("drv-1");
        replayed.ShouldContain("other-node-driver");
    }

    /// <summary>Spawns the host with the subscribing factory, dispatches <paramref name="deploymentId"/> and
    /// waits for the Applied ACK so the child + the initial SubscribeBulk pass are in place.</summary>
    private (IActorRef Actor, Akka.TestKit.TestProbe Coordinator) SpawnHostAndApply(
        IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, IDriverFactory factory)
    {
        var coordinator = CreateTestProbe();
        var actor = Sys.ActorOf(DriverHostActor.Props(
            db, TestNode, coordinator.Ref,
            driverFactory: factory,
            localRoles: new HashSet<string> { "driver" }));

        actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
        coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
        return (actor, coordinator);
    }

    private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
    {
        var probe = CreateTestProbe();
        actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
        return probe.ExpectMsg<NodeDiagnosticsSnapshot>(Timeout);
    }

    /// <summary>Seeds a Sealed v3 deployment: RawFolder "Plant" → DriverInstance "drv-1" (Modbus, ENABLED so a
    /// real child spawns) → Device "dev1" → Tag "speed" (RawPath <c>Plant/Modbus/dev1/speed</c>).</summary>
    private static DeploymentId SeedV3Deployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
        SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
        {
            RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
            DriverInstances = new[]
            {
                new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
            },
            Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
            TagGroups = Array.Empty<object>(),
            Tags = new[]
            {
                new { TagId = "t-speed", DeviceId = "dev-1", TagGroupId = (string?)null, Name = "speed", DataType = "Double", AccessLevel = 1, TagConfig = "{}" },
            },
        }));

    /// <summary>A readable artifact carrying drv-1 AND drv-2, so an apply that genuinely happens is
    /// distinguishable from a short-circuited ACK (which spawns nothing).</summary>
    private static byte[] TwoDriverArtifact() => JsonSerializer.SerializeToUtf8Bytes(new
    {
        RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
        DriverInstances = new[]
        {
            new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
            new { DriverInstanceId = "drv-2", RawFolderId = "rf-plant", Name = "Modbus2", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
        },
        Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
        TagGroups = Array.Empty<object>(),
        Tags = new[]
        {
            new { TagId = "t-speed", DeviceId = "dev-1", TagGroupId = (string?)null, Name = "speed", DataType = "Double", AccessLevel = 1, TagConfig = "{}" },
        },
    });

    /// <summary>Replaces a seeded deployment's artifact in place — the ConfigDb becoming readable again
    /// without the revision changing.</summary>
    private static void SetArtifact(
        IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, byte[] artifact)
    {
        // ArtifactBlob is init-only, so the row is replaced rather than mutated: delete and re-add under
        // the SAME id + revision. Separate SaveChanges calls so the in-memory provider does not see a
        // delete and an insert of the same key in one change set.
        using var ctx = db.CreateDbContext();
        var row = ctx.Deployments.Single(d => d.DeploymentId == deploymentId.Value);
        var rev = row.RevisionHash;
        ctx.Deployments.Remove(row);
        ctx.SaveChanges();

        ctx.Deployments.Add(new Deployment
        {
            DeploymentId = deploymentId.Value,
            RevisionHash = rev,
            Status = DeploymentStatus.Sealed,
            CreatedBy = "test",
            SealedAtUtc = DateTime.UtcNow,
            ArtifactBlob = artifact,
        });
        ctx.SaveChanges();
    }

    /// <summary>Seeds a Sealed deployment whose <c>ArtifactBlob</c> is zero-length — byte-for-byte what the
    /// host's <c>?? Array.Empty&lt;byte&gt;()</c> hands downstream when the row cannot be found.</summary>
    private static DeploymentId SeedUnreadableDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
        SeedDeployment(db, rev, Array.Empty<byte>());

    /// <summary>Seeds a Sealed deployment carrying a real, readable artifact that keeps drv-1 (so its child
    /// survives the reconcile) but declares no tags for it — the driver's desired set becomes empty.</summary>
    private static DeploymentId SeedTaglessDriverDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
        SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
        {
            RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
            DriverInstances = new[]
            {
                new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
            },
            Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
            TagGroups = Array.Empty<object>(),
            Tags = Array.Empty<object>(),
        }));

    /// <summary>Seeds a Sealed deployment carrying a real, readable artifact that declares no drivers.</summary>
    private static DeploymentId SeedDriverlessDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
        SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
        {
            RawFolders = Array.Empty<object>(),
            DriverInstances = Array.Empty<object>(),
            Devices = Array.Empty<object>(),
            TagGroups = Array.Empty<object>(),
            Tags = Array.Empty<object>(),
        }));

    private static DeploymentId SeedDeployment(
        IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev, byte[] artifact)
    {
        var id = DeploymentId.NewId();
        using var ctx = db.CreateDbContext();
        ctx.Deployments.Add(new Deployment
        {
            DeploymentId = id.Value,
            RevisionHash = rev.Value,
            Status = DeploymentStatus.Sealed,
            CreatedBy = "test",
            SealedAtUtc = DateTime.UtcNow,
            ArtifactBlob = artifact,
        });
        ctx.SaveChanges();
        return id;
    }

    /// <summary>Factory producing one shared <see cref="SubscribableStubDriver"/> for the supported type, so a
    /// REAL (non-stubbed) <see cref="DriverInstanceActor"/> child spawns and its subscribe/unsubscribe traffic
    /// is observable.</summary>
    private sealed class SubscribingDriverFactory(string supportedType) : IDriverFactory
    {
        private readonly SubscribableStubDriver _driver = new();

        /// <summary>The reference set passed to the driver's most recent <c>SubscribeAsync</c> call.</summary>
        public IReadOnlyList<string>? LastSubscribedRefs => _driver.LastSubscribedRefs;

        /// <summary>Number of <c>UnsubscribeAsync</c> calls — non-zero means a live handle was torn down.</summary>
        public int UnsubscribeCount => Volatile.Read(ref _driver.UnsubscribeCount);

        /// <summary>Number of <c>ShutdownAsync</c> calls — non-zero means a driver child was stopped.</summary>
        public int ShutdownCount => Volatile.Read(ref _driver.ShutdownCount);

        /// <inheritdoc />
        public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) =>
            string.Equals(driverType, supportedType, StringComparison.Ordinal) ? _driver : null;

        /// <inheritdoc />
        public IReadOnlyCollection<string> SupportedTypes => new[] { supportedType };
    }
}