ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerActorArtifactSecretsTests.cs

using Akka.Actor;
using Akka.TestKit;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
using ZB.MOM.WW.ScadaBridge.TestSupport;
using ZB.MOM.WW.Secrets.Abstractions;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// Site-secrets artifact deploy: <see cref="DeploymentManagerActor.HandleDeployArtifacts"/>
/// writes shipped secrets to <c>deployed_secrets</c>, runs <see cref="DeployedSecretsMirror.ReconcileAsync"/>
/// synchronously, and answers with <see cref="ArtifactDeploymentResponse.SecretsApplied"/> or a failure.
/// Follows the shape of the existing artifact-apply tests in <c>DeploymentManagerActorTests</c>.
/// </summary>
public class DeploymentManagerActorArtifactSecretsTests : TestKit, IDisposable
{
    private readonly SiteStorageService _storage;
    private readonly ScriptCompilationService _compilationService;
    private readonly SharedScriptLibrary _sharedScriptLibrary;
    private readonly TestLocalDb _localDb;

    /// <summary>Initializes a new instance of <c>DeploymentManagerActorArtifactSecretsTests</c>.</summary>
    public DeploymentManagerActorArtifactSecretsTests()
    {
        _localDb = TestLocalDb.CreateTemp("dm-artifact-secrets-test");
        _storage = new SiteStorageService(
            _localDb.Db,
            NullLogger<SiteStorageService>.Instance);
        _storage.InitializeAsync().GetAwaiter().GetResult();
        _compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedScriptLibrary = new SharedScriptLibrary(
            _compilationService, NullLogger<SharedScriptLibrary>.Instance);
    }

    /// <inheritdoc />
    void IDisposable.Dispose()
    {
        Shutdown();
        var path = _localDb.Path;
        _localDb.Dispose();
        TestLocalDb.DeleteFiles(path);
    }

    private IActorRef CreateDeploymentManager(DeployedSecretsMirror? secretsMirror = null, IActorRef? dclManager = null)
    {
        var options = new SiteRuntimeOptions();
        // Positional, not named, past the constructor's first six parameters: a named
        // argument for a trailing optional parameter (secretsMirror) that SKIPS other,
        // still-optional parameters in between is rejected by the compiler inside an
        // expression-tree lambda (Props.Create's Expression<Func<T>> overload) with
        // CS9307 "named argument specification out of position".
        return ActorOf(Props.Create(() => new DeploymentManagerActor(
            _storage,
            _compilationService,
            _sharedScriptLibrary,
            null, // streamManager: no stream manager in tests
            options,
            NullLogger<DeploymentManagerActor>.Instance,
            dclManager,
            null, // healthCollector
            null, // serviceProvider
            null, // loggerFactory
            null, // configFetcher
            null, // startupLoadRetryInterval
            null, // configLoader
            null, // externalSystemCache
            null, // scriptScheduler
            null, // connectionDefinitionLoader
            null, // deployedConfigLoader
            secretsMirror)));
    }

    private DeployedSecretsMirror CreateMirror(FakeSecretStore store, FakeSecretResolver resolver)
    {
        var options = Options.Create(new SiteRuntimeOptions
        {
            DeployedSecretsMirrorInterval = TimeSpan.FromSeconds(15),
        });
        return new DeployedSecretsMirror(
            _storage,
            store,
            resolver,
            invalidator: null,
            options,
            NullLogger<DeployedSecretsMirror>.Instance,
            TimeProvider.System);
    }

    private static DeployArtifactsCommand ArtifactsCommand(
        string deploymentId, IReadOnlyList<SecretArtifact>? secrets) => new(
        DeploymentId: deploymentId,
        SharedScripts: null,
        ExternalSystems: null,
        DatabaseConnections: null,
        NotificationLists: null,
        DataConnections: null,
        SmtpConfigurations: null,
        Timestamp: DateTimeOffset.UtcNow,
        Secrets: secrets);

    private static SecretArtifact Art(string name, long revision, string kek = "kek-1") => new(
        Name: name,
        ContentType: "Text",
        Ciphertext: [1, 2, 3, 4, 5],
        Nonce: [10, 11, 12],
        Tag: [20, 21, 22, 23],
        WrappedDek: [30, 31, 32, 33, 34, 35],
        WrapNonce: [40, 41, 42],
        WrapTag: [50, 51, 52, 53],
        KekId: kek,
        Revision: revision,
        CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero),
        UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero),
        Description: "test secret");

    /// <summary>Verifies that secrets null leaves table untouched and secrets applied null.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Secrets_Null_LeavesTableUntouched_AndSecretsAppliedNull()
    {
        var store = new FakeSecretStore();
        var resolver = new FakeSecretResolver();
        var mirror = CreateMirror(store, resolver);
        var manager = CreateDeploymentManager(mirror);

        manager.Tell(ArtifactsCommand("dep-null", secrets: null));

        var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        Assert.Null(response.SecretsApplied);

        var rows = await _storage.GetDeployedSecretsAsync();
        Assert.Empty(rows);
    }

    /// <summary>Verifies that secrets present full set write sync reconcile secrets applied equals shipped count.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Secrets_Present_FullSetWrite_SyncReconcile_SecretsAppliedEqualsShippedCount()
    {
        var store = new FakeSecretStore();
        var resolver = new FakeSecretResolver();
        var mirror = CreateMirror(store, resolver);
        var manager = CreateDeploymentManager(mirror);

        manager.Tell(ArtifactsCommand("dep-2secrets", secrets: [Art("a", 1), Art("b", 1)]));

        var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        Assert.Equal(2, response.SecretsApplied);
        Assert.Null(response.ErrorMessage);

        // Reconcile ran synchronously before the reply — both secrets already applied.
        Assert.True(store.Contains("a"));
        Assert.True(store.Contains("b"));
    }

    /// <summary>Verifies that secrets empty with mirror secrets applied zero success.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Secrets_Empty_WithMirror_SecretsAppliedZero_Success()
    {
        var store = new FakeSecretStore();
        var resolver = new FakeSecretResolver();
        var mirror = CreateMirror(store, resolver);
        var manager = CreateDeploymentManager(mirror);

        manager.Tell(ArtifactsCommand("dep-empty", secrets: []));

        var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        Assert.Equal(0, response.SecretsApplied);
    }

    /// <summary>Verifies that secrets dropped reply names the retired secrets.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Secrets_Dropped_ReplyNamesTheRetiredSecrets()
    {
        var store = new FakeSecretStore();
        var resolver = new FakeSecretResolver();
        var mirror = CreateMirror(store, resolver);
        var manager = CreateDeploymentManager(mirror);

        manager.Tell(ArtifactsCommand("dep-1", secrets: [Art("a", 1), Art("b", 1)]));
        var first = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.True(first.Success);
        Assert.Null(first.SecretsRemoved);

        // The next deploy no longer references "b" (a removal-only change for that name):
        // the reply names it so central can audit the removal.
        manager.Tell(ArtifactsCommand("dep-2", secrets: [Art("a", 1)]));
        var second = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.True(second.Success);
        Assert.Equal(["b"], second.SecretsRemoved!);

        // Dropping the last reference retires "a" the same way.
        manager.Tell(ArtifactsCommand("dep-3", secrets: []));
        var third = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.True(third.Success);
        Assert.Equal(0, third.SecretsApplied);
        Assert.Equal(["a"], third.SecretsRemoved!);
        await Task.CompletedTask;
    }

    /// <summary>Verifies that secrets mirror fails one success false kek id in message secrets applied excludes failure.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Secrets_MirrorFailsOne_SuccessFalse_KekIdInMessage_SecretsAppliedExcludesFailure()
    {
        var store = new FakeSecretStore();
        var resolver = new FakeSecretResolver();
        resolver.ThrowFor("bad");
        var mirror = CreateMirror(store, resolver);
        var manager = CreateDeploymentManager(mirror);

        manager.Tell(ArtifactsCommand(
            "dep-onefails", secrets: [Art("good", 1), Art("bad", 1, kek: "kek-central")]));

        var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.False(response.Success);
        Assert.NotNull(response.ErrorMessage);
        Assert.Contains("kek-central", response.ErrorMessage);
        Assert.Contains("'bad'", response.ErrorMessage);
        Assert.Equal(1, response.SecretsApplied); // "good" applied+verified; "bad" failed
    }

    /// <summary>Verifies that secrets non empty no mirror registered fails deploy.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Secrets_NonEmpty_NoMirrorRegistered_FailsDeploy()
    {
        var manager = CreateDeploymentManager(secretsMirror: null);

        manager.Tell(ArtifactsCommand("dep-nomirror", secrets: [Art("a", 1)]));

        var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.False(response.Success);
        Assert.Equal("Secret deployment is not enabled on this site node", response.ErrorMessage);
    }

    /// <summary>Verifies that secrets empty no mirror registered treated as zero success.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Secrets_Empty_NoMirrorRegistered_TreatedAsZeroSuccess()
    {
        var manager = CreateDeploymentManager(secretsMirror: null);

        manager.Tell(ArtifactsCommand("dep-nomirror-empty", secrets: []));

        var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        Assert.Equal(0, response.SecretsApplied);
    }

    /// <summary>Verifies that secrets removal failure does not fail deploy and excluded from error message.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Secrets_RemovalFailure_DoesNotFailDeploy_AndExcludedFromErrorMessage()
    {
        // A row that was previously mirrored (ledgered) but is now tombstoned upstream, and
        // whose removal fails on this node's store — a failure UNRELATED to the names shipped
        // by THIS deploy. Per ruling: log at Warning, never surface in ErrorMessage, never fail
        // the deploy, never affect SecretsApplied (which only counts SHIPPED names).
        var store = new FakeSecretStore();
        var resolver = new FakeSecretResolver();
        var mirror = CreateMirror(store, resolver);
        var manager = CreateDeploymentManager(mirror);

        // First deploy ships + ledgers "gone".
        manager.Tell(ArtifactsCommand("dep-r1", secrets: [Art("gone", 1)]));
        var r1 = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.True(r1.Success);
        Assert.True(store.Contains("gone"));

        // Make the store's delete fail on the next pass, then ship a deploy that no longer
        // references "gone" (and ships an unrelated, healthy secret).
        store.ThrowOnDelete = true;
        manager.Tell(ArtifactsCommand("dep-r2", secrets: [Art("kept", 1)]));
        var r2 = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));

        Assert.True(r2.Success);
        Assert.Equal(1, r2.SecretsApplied); // only "kept" is shipped by this deploy
        Assert.Null(r2.ErrorMessage);
    }

    /// <summary>Verifies that data connections pushed to dcl only after shipped secrets are mirrored.</summary>
    [Fact]
    public void DataConnections_PushedToDcl_OnlyAfterShippedSecretsAreMirrored()
    {
        // A connection whose config references a secret shipped by THIS
        // deploy must find it on its first connect, so the DCL create must not reach the
        // DCL manager before the secret is in this node's store.
        var store = new FakeSecretStore();
        var resolver = new FakeSecretResolver();
        var mirror = CreateMirror(store, resolver);
        var dcl = CreateTestProbe();
        bool? secretPresentWhenDclSawCreate = null;
        dcl.SetAutoPilot(new DelegateAutoPilot((_, msg) =>
        {
            if (msg is ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection.CreateConnectionCommand)
            {
                secretPresentWhenDclSawCreate ??= store.Contains("gw-key");
            }

            return AutoPilot.KeepRunning;
        }));
        var manager = CreateDeploymentManager(mirror, dcl.Ref);

        var command = ArtifactsCommand("dep-order", secrets: [Art("gw-key", 1)]) with
        {
            DataConnections = [new DataConnectionArtifact("Gw", "OpcUa", "{}", null, 3)],
        };
        manager.Tell(command);

        var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        Assert.Equal(1, response.SecretsApplied);
        AwaitCondition(() => secretPresentWhenDclSawCreate.HasValue, TimeSpan.FromSeconds(10));
        Assert.True(secretPresentWhenDclSawCreate);
    }

    /// <summary>Verifies that data connections still stored and pushed when secrets step fails.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task DataConnections_StillStoredAndPushed_WhenSecretsStepFails()
    {
        // The reorder must not lose the connection push when the secrets step fails.
        var dcl = CreateTestProbe();
        var manager = CreateDeploymentManager(secretsMirror: null, dclManager: dcl.Ref);

        var command = ArtifactsCommand("dep-order-fail", secrets: [Art("gw-key", 1)]) with
        {
            DataConnections = [new DataConnectionArtifact("Gw", "OpcUa", "{}", null, 3)],
        };
        manager.Tell(command);

        var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
        Assert.False(response.Success);
        dcl.ExpectMsg<ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection.CreateConnectionCommand>(
            TimeSpan.FromSeconds(10));
        Assert.Contains(await _storage.GetAllDataConnectionDefinitionsAsync(), d => d.Name == "Gw");
    }

    // ── Fakes (mirrors DeployedSecretsMirrorTests' fakes; kept local/minimal per-task) ──

    private sealed class FakeSecretStore : ISecretStore
    {
        private readonly Dictionary<string, StoredSecret> _rows = new(StringComparer.Ordinal);

        /// <summary>When set, deleting any secret throws.</summary>
        public bool ThrowOnDelete { get; set; }

        /// <summary>Returns whether the store currently holds a live secret with the given name.</summary>
        /// <param name="name">The secret name.</param>
        /// <returns>True when a live secret with that name exists.</returns>
        public bool Contains(string name) => _rows.TryGetValue(name, out var row) && !row.IsDeleted;

        /// <summary>Returns the stored or configured secret.</summary>
        /// <param name="name">The secret name.</param>
        /// <param name="ct">The cancellation token.</param>
        /// <returns>The secret, or null when absent.</returns>
        public Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct) =>
            Task.FromResult(_rows.TryGetValue(name.Value, out var row) ? row : null);

        /// <summary>Stores the row, replacing any existing row of the same name.</summary>
        /// <param name="row">The secret row.</param>
        /// <param name="ct">The cancellation token.</param>
        /// <returns>A task that represents the asynchronous operation.</returns>
        public Task UpsertAsync(StoredSecret row, CancellationToken ct)
        {
            _rows[row.Name.Value] = row;
            return Task.CompletedTask;
        }

        /// <summary>Soft-deletes the named secret.</summary>
        /// <param name="name">The secret name.</param>
        /// <param name="actor">The acting user.</param>
        /// <param name="ct">The cancellation token.</param>
        /// <returns>True when a live secret was deleted; otherwise false.</returns>
        public Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct)
        {
            if (ThrowOnDelete)
            {
                throw new InvalidOperationException("simulated delete fault");
            }

            if (!_rows.TryGetValue(name.Value, out var existing) || existing.IsDeleted)
            {
                return Task.FromResult(false);
            }

            _rows[name.Value] = existing with
            {
                IsDeleted = true,
                DeletedUtc = DateTimeOffset.UtcNow,
                UpdatedUtc = DateTimeOffset.UtcNow,
                Revision = existing.Revision + 1,
                UpdatedBy = actor,
            };
            return Task.FromResult(true);
        }

        /// <summary>Returns an empty metadata list.</summary>
        /// <param name="includeDeleted">Whether deleted secrets are included.</param>
        /// <param name="ct">The cancellation token.</param>
        /// <returns>An empty list.</returns>
        public Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct) =>
            Task.FromResult<IReadOnlyList<SecretMetadata>>([]);

        /// <summary>Returns an empty manifest.</summary>
        /// <param name="ct">The cancellation token.</param>
        /// <returns>An empty manifest.</returns>
        public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
            Task.FromResult<IReadOnlyList<SecretManifestEntry>>([]);

        /// <summary>Applies a replicated row when it is newer than the stored one.</summary>
        /// <param name="row">The replicated row.</param>
        /// <param name="ct">The cancellation token.</param>
        /// <returns>A task that represents the asynchronous operation.</returns>
        public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct)
        {
            if (_rows.TryGetValue(row.Name.Value, out var existing)
                && !SecretLastWriterWins.IsNewer(row.UpdatedUtc, row.Revision, existing.UpdatedUtc, existing.Revision))
            {
                return Task.CompletedTask;
            }

            _rows[row.Name.Value] = row;
            return Task.CompletedTask;
        }

        /// <summary>Reports that no rewrap was applied.</summary>
        /// <param name="rewrappedRow">The rewrapped row.</param>
        /// <param name="expectedCurrentWrappedDek">The wrapped key the row is expected to hold.</param>
        /// <param name="ct">The cancellation token.</param>
        /// <returns>Always false.</returns>
        public Task<bool> ApplyRewrapAsync(
            StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
            Task.FromResult(false);
    }

    private sealed class FakeSecretResolver : ISecretResolver
    {
        private readonly HashSet<string> _throwing = new(StringComparer.Ordinal);

        /// <summary>Throws the configured fault when the named secret is requested.</summary>
        /// <param name="name">The secret name that should fail.</param>
        public void ThrowFor(string name) => _throwing.Add(name);

        /// <summary>Returns the stored or configured secret.</summary>
        /// <param name="name">The secret name.</param>
        /// <param name="ct">The cancellation token.</param>
        /// <returns>The secret, or null when absent.</returns>
        public Task<string?> GetAsync(SecretName name, CancellationToken ct)
        {
            if (_throwing.Contains(name.Value))
            {
                throw new InvalidOperationException("simulated decrypt failure");
            }

            return Task.FromResult<string?>("decrypted-value");
        }
    }
}