scadaproj/ZB.MOM.WW.Secrets/tests/ZB.MOM.WW.Secrets.Replicator.Grpc.Tests/SecretsHubIntegrationTests.cs

using System.Diagnostics;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Replicator.Grpc.V1;

namespace ZB.MOM.WW.Secrets.Replicator.Grpc.Tests;

/// <summary>
/// The gRPC secrets hub end to end: a central node hosting the real service through its own DI and
/// interceptor, a site node running the real client and sweep through its own DI, and a real gRPC
/// call between them.
/// </summary>
/// <remarks>
/// <para>
/// Everything the unit suites necessarily substitute is real here — the registration extensions, the
/// generated client and service, the fail-closed interceptor as the container actually activates it,
/// protobuf encoding on both sides, and two real SQLite stores. What that buys is the class of defect
/// no unit test can reach: pieces that are each correct and are never connected to one another.
/// </para>
/// <para>
/// <b>No credential in this file is a real one.</b> The two token values are fabricated for these
/// tests, and one of the tests exists to assert that neither ever reaches a log sink on either side.
/// </para>
/// </remarks>
public sealed class SecretsHubIntegrationTests : IAsyncLifetime
{
    private const string ConfiguredToken = "central-hub-credential-7f3a";
    private const string WrongToken = "impostor-credential-91cd";

    private static readonly TimeSpan Patience = TimeSpan.FromSeconds(10);

    private readonly TempSecretStore _hubStore = new("hub");
    private readonly TempSecretStore _localStore = new("site");
    private readonly List<IAsyncDisposable> _disposables = [];

    /// <inheritdoc />
    public Task InitializeAsync() => Task.CompletedTask;

    /// <inheritdoc />
    public async Task DisposeAsync()
    {
        foreach (IAsyncDisposable disposable in _disposables)
        {
            await disposable.DisposeAsync();
        }

        _localStore.Dispose();
        _hubStore.Dispose();
    }

    // ---- 1. end-to-end convergence ----------------------------------------------------------

    [Fact]
    public async Task One_sweep_carries_every_row_across_the_wire_byte_identically()
    {
        // The four rows between them cover every shape the wire has to keep distinguishable: a fully
        // populated row with non-UTC offsets on both timestamps, a row whose every nullable field is
        // null, a row whose nullable strings are deliberately EMPTY rather than absent, and a
        // tombstone. Empty-versus-null is the one a plain proto3 string would silently collapse.
        StoredSecret populated = Row("db/central", 0xA1) with
        {
            Description = "central connection string",
            ContentType = SecretContentType.ConnectionString,
            Revision = 7,
            CreatedUtc = new DateTimeOffset(2026, 1, 2, 3, 4, 5, 678, TimeSpan.FromHours(-5)).AddTicks(9012),
            UpdatedUtc = new DateTimeOffset(2026, 8, 7, 9, 10, 11, 121, TimeSpan.FromHours(5.5)).AddTicks(3456),
            CreatedBy = "alice",
            UpdatedBy = "bob",
        };
        StoredSecret nulls = Row("api/token", 0xB2);
        StoredSecret empties = Row("svc/empties", 0xC3) with
        {
            Description = string.Empty,
            CreatedBy = string.Empty,
            UpdatedBy = string.Empty,
        };
        StoredSecret tombstone = Row("retired/legacy", 0xD4) with
        {
            IsDeleted = true,
            DeletedUtc = new DateTimeOffset(2026, 8, 7, 9, 30, 0, TimeSpan.FromHours(9)),
            Revision = 3,
            UpdatedBy = "carol",
        };

        await SeedHubAsync(populated, nulls, empties, tombstone);

        SecretsHubTestClient client = await ConnectAsync();

        int pulled = await client.Sweeper.SweepAsync(CancellationToken.None);

        Assert.Equal(4, pulled);

        foreach (StoredSecret expected in new[] { populated, nulls, empties, tombstone })
        {
            StoredSecret hubRow = await RequireAsync(_hubStore.Store, expected.Name);
            StoredSecret localRow = await RequireAsync(_localStore.Store, expected.Name);

            AssertIdentical(hubRow, localRow);
        }
    }

    [Fact]
    public async Task Convergence_evicts_the_resolver_cache_for_exactly_the_rows_it_pulled()
    {
        await SeedHubAsync(Row("db/central", 0xA1), Row("api/token", 0xB2));
        await _localStore.Store.ApplyReplicatedAsync(Row("only/local", 0x11), CancellationToken.None);

        SecretsHubTestClient client = await ConnectAsync();

        await client.Sweeper.SweepAsync(CancellationToken.None);

        Assert.Equal(
            ["api/token", "db/central"],
            client.Invalidated.Select(name => name.Value).Order(StringComparer.Ordinal));
    }

    [Fact]
    public async Task A_converged_follower_pulls_nothing_on_the_next_sweep()
    {
        await SeedHubAsync(Row("db/central", 0xA1));

        SecretsHubTestClient client = await ConnectAsync();
        await client.Sweeper.SweepAsync(CancellationToken.None);

        Assert.Equal(0, await client.Sweeper.SweepAsync(CancellationToken.None));
    }

    // ---- 2. delete propagation --------------------------------------------------------------

    [Fact]
    public async Task A_hub_side_tombstone_replaces_a_live_local_row()
    {
        StoredSecret live = Row("api/token", 0xB2) with
        {
            Revision = 1,
            CreatedUtc = DateTimeOffset.UtcNow.AddDays(-1),
            UpdatedUtc = DateTimeOffset.UtcNow.AddDays(-1),
        };

        await SeedHubAsync(live);
        await _localStore.Store.ApplyReplicatedAsync(live, CancellationToken.None);

        SecretsHubTestClient client = await ConnectAsync();

        // Nothing to do yet: both sides hold the identical live row.
        Assert.Equal(0, await client.Sweeper.SweepAsync(CancellationToken.None));

        // A real delete on the hub — higher revision, newer timestamp, tombstone flag set.
        Assert.True(await _hubStore.Store.DeleteAsync(live.Name, "admin", CancellationToken.None));

        Assert.Equal(1, await client.Sweeper.SweepAsync(CancellationToken.None));

        StoredSecret hubRow = await RequireAsync(_hubStore.Store, live.Name);
        StoredSecret localRow = await RequireAsync(_localStore.Store, live.Name);

        Assert.True(localRow.IsDeleted);
        Assert.NotNull(localRow.DeletedUtc);
        AssertIdentical(hubRow, localRow);
    }

    // ---- 3. hub outage at boot, then recovery -----------------------------------------------

    [Fact]
    public async Task A_hub_that_is_down_at_boot_leaves_the_last_known_good_store_intact_and_converges_on_recovery()
    {
        StoredSecret lastKnownGood = Row("db/central", 0x77) with
        {
            Revision = 1,
            CreatedUtc = DateTimeOffset.UtcNow.AddDays(-2),
            UpdatedUtc = DateTimeOffset.UtcNow.AddDays(-2),
        };
        await _localStore.Store.ApplyReplicatedAsync(lastKnownGood, CancellationToken.None);

        StoredSecret newer = lastKnownGood with
        {
            Ciphertext = [0x88],
            Revision = 2,
            UpdatedUtc = DateTimeOffset.UtcNow,
        };
        await SeedHubAsync(newer);

        // The hub exists, but this node cannot reach it: the transport is disconnected before the
        // node boots, which is the ordinary "central is down while a site restarts" case.
        var transport = new SwitchableHttpHandler();
        SecretsHubTestServer server = await StartHubAsync();
        SecretsHubTestClient client = Track(
            SecretsHubTestClient.Create(_localStore.Store, transport, ConfiguredToken));

        await client.Sweeper.StartAsync(CancellationToken.None);

        await WaitForAsync(
            () => client.Logs.WarningsFrom<GrpcSecretSyncService>().Count > 0,
            "the startup sweep never reported the outage");

        // The node did not crash, and it is still serving exactly what it had.
        Assert.NotNull(client.Sweeper.ExecuteTask);
        Assert.False(client.Sweeper.ExecuteTask!.IsFaulted);
        AssertIdentical(lastKnownGood, await RequireAsync(_localStore.Store, lastKnownGood.Name));

        // Central comes back.
        transport.ConnectTo(server.CreateHandler());

        Assert.Equal(1, await client.Sweeper.SweepAsync(CancellationToken.None));
        AssertIdentical(
            await RequireAsync(_hubStore.Store, newer.Name),
            await RequireAsync(_localStore.Store, newer.Name));

        await client.Sweeper.StopAsync(CancellationToken.None);
        Assert.False(client.Sweeper.ExecuteTask!.IsFaulted);
    }

    [Fact]
    public async Task An_outage_costs_exactly_one_warning_and_no_row_ever_disappears()
    {
        // The failure mode this guards is not "the sweep failed" — it is a hub that answers "I have
        // nothing" being indistinguishable from a hub that does not answer, which would empty the
        // follower's store. A sweep against a dead transport must move nothing at all.
        StoredSecret held = Row("api/token", 0x55);
        await _localStore.Store.ApplyReplicatedAsync(held, CancellationToken.None);

        var transport = new SwitchableHttpHandler();
        SecretsHubTestClient client = Track(
            SecretsHubTestClient.Create(_localStore.Store, transport, ConfiguredToken));

        await client.Sweeper.StartAsync(CancellationToken.None);
        await WaitForAsync(
            () => client.Logs.WarningsFrom<GrpcSecretSyncService>().Count > 0,
            "the startup sweep never reported the outage");
        await client.Sweeper.StopAsync(CancellationToken.None);

        Assert.Single(client.Logs.WarningsFrom<GrpcSecretSyncService>());
        AssertIdentical(held, await RequireAsync(_localStore.Store, held.Name));
    }

    [Fact]
    public async Task A_transport_failure_reaches_the_raw_sweep_as_a_fault_rather_than_an_empty_answer()
    {
        // The containment above is only meaningful because the underlying call really does fail.
        var transport = new SwitchableHttpHandler();
        SecretsHubTestClient client = Track(
            SecretsHubTestClient.Create(
                _localStore.Store, transport, ConfiguredToken, syncOnStartup: false));

        await Assert.ThrowsAsync<RpcException>(
            () => client.Sweeper.SweepAsync(CancellationToken.None));
    }

    // ---- 4. authentication, fail-closed, over the wire ---------------------------------------

    [Fact]
    public async Task A_wrong_bearer_token_is_refused()
    {
        await SeedHubAsync(Row("db/central", 0xA1));
        SecretsHubTestServer server = await StartHubAsync();
        SecretsHubTestClient client = Track(
            SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), WrongToken));

        RpcException error = await Assert.ThrowsAsync<RpcException>(
            () => client.Hub.GetManifestAsync(CancellationToken.None));

        Assert.Equal(StatusCode.Unauthenticated, error.StatusCode);
        Assert.Null(await _localStore.Store.GetAsync(new SecretName("db/central"), CancellationToken.None));
    }

    [Fact]
    public async Task A_call_with_no_credential_at_all_is_refused_in_exactly_the_same_words()
    {
        // Uniform denial, asserted where it matters: across the wire, between two DIFFERENT failure
        // modes. A caller that could tell "no token" from "wrong token" would have a free oracle.
        SecretsHubTestServer server = await StartHubAsync();
        using GrpcChannel channel = ChannelOver(server.CreateHandler());
        var bare = new SecretsHub.SecretsHubClient(channel);

        RpcException noHeader = await Assert.ThrowsAsync<RpcException>(
            async () => await bare.GetManifestAsync(new GetManifestRequest()));

        SecretsHubTestClient wrong = Track(
            SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), WrongToken));
        RpcException wrongToken = await Assert.ThrowsAsync<RpcException>(
            () => wrong.Hub.GetManifestAsync(CancellationToken.None));

        Assert.Equal(StatusCode.Unauthenticated, noHeader.StatusCode);
        Assert.Equal(wrongToken.StatusCode, noHeader.StatusCode);
        Assert.Equal(wrongToken.Status.Detail, noHeader.Status.Detail);
    }

    [Fact]
    public async Task A_hub_with_no_token_configured_refuses_even_a_well_formed_credential()
    {
        SecretsHubTestServer server = Track(
            await SecretsHubTestServer.StartWithNoTokenConfiguredAsync(_hubStore.Store));
        SecretsHubTestClient client = Track(
            SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken));

        RpcException error = await Assert.ThrowsAsync<RpcException>(
            () => client.Hub.GetManifestAsync(CancellationToken.None));

        Assert.Equal(StatusCode.Unauthenticated, error.StatusCode);
    }

    [Fact]
    public async Task An_unconfigured_hub_warns_once_however_many_callers_it_refuses()
    {
        // Once per process, not once per call — and the interceptor can only honour that if the
        // container hands gRPC the singleton it registered. That wiring is exactly what a unit test
        // constructing the interceptor by hand cannot check.
        SecretsHubTestServer server = Track(
            await SecretsHubTestServer.StartWithNoTokenConfiguredAsync(_hubStore.Store));
        SecretsHubTestClient client = Track(
            SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken));

        for (int attempt = 0; attempt < 3; attempt++)
        {
            await Assert.ThrowsAsync<RpcException>(
                () => client.Hub.GetManifestAsync(CancellationToken.None));
        }

        // Scoped to the CONFIGURATION warning, because the interceptor deliberately emits a second
        // kind on this path: a rate-limited summary of the denials themselves. Both are warnings
        // from the same category, so an unfiltered Single would now be counting two signals as one.
        IReadOnlyList<CapturedLog> warnings = server.Logs.WarningsFrom<SecretsHubAuthInterceptor>();
        CapturedLog warning = Assert.Single(
            warnings, w => w.Message.Contains("no bearer token configured", StringComparison.Ordinal));
        Assert.Contains("Secrets:GrpcHub:BearerToken", warning.Message, StringComparison.Ordinal);

        // And the denial summary is likewise once, not once per refused caller — the point of the
        // rate limit, checked here through the real registration rather than a hand-built instance.
        Assert.Single(
            warnings, w => w.Message.Contains("The gRPC secrets hub denied", StringComparison.Ordinal));
    }

    [Fact]
    public async Task No_log_line_on_either_side_carries_credential_material()
    {
        await SeedHubAsync(Row("db/central", 0xA1));
        SecretsHubTestServer server = await StartHubAsync();

        SecretsHubTestClient accepted = Track(
            SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken));
        await accepted.Sweeper.SweepAsync(CancellationToken.None);

        SecretsHubTestClient refused = Track(
            SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), WrongToken));
        await Assert.ThrowsAsync<RpcException>(
            () => refused.Hub.GetManifestAsync(CancellationToken.None));

        IEnumerable<CapturedLog> everything = server.Logs.Entries
            .Concat(accepted.Logs.Entries)
            .Concat(refused.Logs.Entries);

        // Captured at Trace on both hosts, so this covers the most verbose setting a deployment
        // could switch on — the configured token, the presented one, and the header that carried it.
        Assert.NotEmpty(everything);
        foreach (CapturedLog entry in everything)
        {
            Assert.DoesNotContain(ConfiguredToken, entry.AllText, StringComparison.OrdinalIgnoreCase);
            Assert.DoesNotContain(WrongToken, entry.AllText, StringComparison.OrdinalIgnoreCase);
        }
    }

    // ---- 5. the request cap, over the wire ---------------------------------------------------

    [Fact]
    public async Task A_fetch_over_the_configured_cap_is_refused_as_resource_exhausted()
    {
        await SeedHubAsync(Row("db/central", 0xA1), Row("api/token", 0xB2), Row("svc/empties", 0xC3));
        SecretsHubTestServer server = await StartHubAsync(maxNamesPerRequest: 2);
        SecretsHubTestClient client = Track(
            SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken));

        RpcException error = await Assert.ThrowsAsync<RpcException>(
            () => client.Hub.GetManyAsync(
                [new SecretName("db/central"), new SecretName("api/token"), new SecretName("svc/empties")],
                CancellationToken.None));

        Assert.Equal(StatusCode.ResourceExhausted, error.StatusCode);
    }

    [Fact]
    public async Task A_fetch_exactly_at_the_cap_is_served()
    {
        await SeedHubAsync(Row("db/central", 0xA1), Row("api/token", 0xB2));
        SecretsHubTestServer server = await StartHubAsync(maxNamesPerRequest: 2);
        SecretsHubTestClient client = Track(
            SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken));

        IReadOnlyList<StoredSecret> rows = await client.Hub.GetManyAsync(
            [new SecretName("db/central"), new SecretName("api/token")], CancellationToken.None);

        Assert.Equal(2, rows.Count);
    }

    // ---- 6. transport faults stay distinguishable from parse faults --------------------------

    [Fact]
    public async Task An_RpcException_from_the_call_invoker_escapes_the_hub_client_unchanged()
    {
        // The client catches PARSE failures per item and drops them; it must never catch a transport
        // failure. If it did, an unreachable hub would be indistinguishable from an empty one, and
        // the sweep would have no way to tell "central said nothing" from "central has nothing".
        var thrown = new RpcException(new Status(StatusCode.Unavailable, "hub unreachable"));
        var invoker = new FakeSecretsHubCallInvoker { FaultWith = () => thrown };
        GrpcSecretsHubClient client = ClientOver(invoker);

        RpcException fromManifest = await Assert.ThrowsAsync<RpcException>(
            () => client.GetManifestAsync(CancellationToken.None));
        RpcException fromFetch = await Assert.ThrowsAsync<RpcException>(
            () => client.GetManyAsync([new SecretName("db/central")], CancellationToken.None));

        Assert.Same(thrown, fromManifest);
        Assert.Same(thrown, fromFetch);
    }

    // ---- helpers -----------------------------------------------------------------------------

    private static GrpcSecretsHubClient ClientOver(CallInvoker invoker) => new(
        new SecretsHub.SecretsHubClient(invoker),
        Options.Create(new SecretsGrpcHubClientOptions
        {
            Endpoint = "http://central.test",
            BearerToken = "unused-by-the-invoker",
        }),
        new RecordingLogger<GrpcSecretsHubClient>());

    private static GrpcChannel ChannelOver(HttpMessageHandler handler) =>
        GrpcChannel.ForAddress("http://central.test", new GrpcChannelOptions { HttpHandler = handler });

    private async Task<SecretsHubTestServer> StartHubAsync(int? maxNamesPerRequest = null) =>
        Track(await SecretsHubTestServer.StartAsync(
            _hubStore.Store, ConfiguredToken, maxNamesPerRequest));

    private async Task<SecretsHubTestClient> ConnectAsync()
    {
        SecretsHubTestServer server = await StartHubAsync();

        return Track(SecretsHubTestClient.Create(
            _localStore.Store, server.CreateHandler(), ConfiguredToken, syncOnStartup: false));
    }

    private T Track<T>(T disposable) where T : IAsyncDisposable
    {
        _disposables.Add(disposable);
        return disposable;
    }

    // Seeded through ApplyReplicatedAsync rather than UpsertAsync: the upsert path stamps its own
    // timestamps and revision, which would erase the exact values these tests are about.
    private async Task SeedHubAsync(params StoredSecret[] rows)
    {
        foreach (StoredSecret row in rows)
        {
            await _hubStore.Store.ApplyReplicatedAsync(row, CancellationToken.None);
        }
    }

    private static async Task<StoredSecret> RequireAsync(ISecretStore store, SecretName name)
    {
        StoredSecret? row = await store.GetAsync(name, CancellationToken.None);

        Assert.True(row is not null, $"expected the store to hold '{name.Value}'");
        return row!;
    }

    private static async Task WaitForAsync(Func<bool> condition, string because)
    {
        var elapsed = Stopwatch.StartNew();

        while (!condition() && elapsed.Elapsed < Patience)
        {
            await Task.Delay(10);
        }

        Assert.True(condition(), because);
    }

    /// <summary>
    /// Field-by-field equality of two rows — every byte array, every timestamp, every nullable.
    /// </summary>
    /// <remarks>
    /// Written out rather than delegated to record equality on purpose: <see cref="StoredSecret"/>
    /// holds <c>byte[]</c>, which records compare by REFERENCE, so <c>Assert.Equal(a, b)</c> on the
    /// whole row would pass only for the same instance and silently prove nothing about a row that
    /// crossed a wire. The byte-identical claim in the live gate is exactly this assertion.
    /// </remarks>
    private static void AssertIdentical(StoredSecret expected, StoredSecret actual)
    {
        Assert.Equal(expected.Name.Value, actual.Name.Value);
        Assert.Equal(expected.Description, actual.Description);
        Assert.Equal(expected.ContentType, actual.ContentType);
        Assert.Equal(expected.Ciphertext, actual.Ciphertext);
        Assert.Equal(expected.Nonce, actual.Nonce);
        Assert.Equal(expected.Tag, actual.Tag);
        Assert.Equal(expected.WrappedDek, actual.WrappedDek);
        Assert.Equal(expected.WrapNonce, actual.WrapNonce);
        Assert.Equal(expected.WrapTag, actual.WrapTag);
        Assert.Equal(expected.KekId, actual.KekId);
        Assert.Equal(expected.Revision, actual.Revision);
        Assert.Equal(expected.IsDeleted, actual.IsDeleted);
        Assert.Equal(expected.DeletedUtc, actual.DeletedUtc);
        Assert.Equal(expected.CreatedUtc, actual.CreatedUtc);
        Assert.Equal(expected.UpdatedUtc, actual.UpdatedUtc);
        Assert.Equal(expected.CreatedBy, actual.CreatedBy);
        Assert.Equal(expected.UpdatedBy, actual.UpdatedBy);

        // Offset, not just instant: DateTimeOffset equality is instant-only, so a wire form that
        // normalized every timestamp to UTC would satisfy the assertions above while quietly
        // discarding the offset the store persisted.
        Assert.Equal(expected.CreatedUtc.Offset, actual.CreatedUtc.Offset);
        Assert.Equal(expected.UpdatedUtc.Offset, actual.UpdatedUtc.Offset);
        Assert.Equal(expected.DeletedUtc?.Offset, actual.DeletedUtc?.Offset);
    }

    private static StoredSecret Row(string name, byte marker) => new()
    {
        Name = new SecretName(name),
        ContentType = SecretContentType.Text,
        Ciphertext = [marker],
        Nonce = [1, 2],
        Tag = [3, 4, 5],
        WrappedDek = [6, 7, 8, 9],
        WrapNonce = [10],
        WrapTag = [11, 12],
        KekId = "sha256:kek-under-test",
        Revision = 0,
        CreatedUtc = new DateTimeOffset(2026, 8, 7, 1, 2, 3, TimeSpan.Zero),
        UpdatedUtc = new DateTimeOffset(2026, 8, 7, 1, 2, 3, TimeSpan.Zero),
    };
}