using System.Diagnostics; using Grpc.Core; using Grpc.Net.Client; using Microsoft.Extensions.Options; using ZB.MOM.WW.Secrets.Abstractions; using ZB.MOM.WW.Secrets.Replicator.Grpc.V1; namespace ZB.MOM.WW.Secrets.Replicator.Grpc.Tests; /// /// The gRPC secrets hub end to end: a central node hosting the real service through its own DI and /// interceptor, a site node running the real client and sweep through its own DI, and a real gRPC /// call between them. /// /// /// /// Everything the unit suites necessarily substitute is real here — the registration extensions, the /// generated client and service, the fail-closed interceptor as the container actually activates it, /// protobuf encoding on both sides, and two real SQLite stores. What that buys is the class of defect /// no unit test can reach: pieces that are each correct and are never connected to one another. /// /// /// No credential in this file is a real one. The two token values are fabricated for these /// tests, and one of the tests exists to assert that neither ever reaches a log sink on either side. /// /// public sealed class SecretsHubIntegrationTests : IAsyncLifetime { private const string ConfiguredToken = "central-hub-credential-7f3a"; private const string WrongToken = "impostor-credential-91cd"; private static readonly TimeSpan Patience = TimeSpan.FromSeconds(10); private readonly TempSecretStore _hubStore = new("hub"); private readonly TempSecretStore _localStore = new("site"); private readonly List _disposables = []; /// public Task InitializeAsync() => Task.CompletedTask; /// public async Task DisposeAsync() { foreach (IAsyncDisposable disposable in _disposables) { await disposable.DisposeAsync(); } _localStore.Dispose(); _hubStore.Dispose(); } // ---- 1. end-to-end convergence ---------------------------------------------------------- [Fact] public async Task One_sweep_carries_every_row_across_the_wire_byte_identically() { // The four rows between them cover every shape the wire has to keep distinguishable: a fully // populated row with non-UTC offsets on both timestamps, a row whose every nullable field is // null, a row whose nullable strings are deliberately EMPTY rather than absent, and a // tombstone. Empty-versus-null is the one a plain proto3 string would silently collapse. StoredSecret populated = Row("db/central", 0xA1) with { Description = "central connection string", ContentType = SecretContentType.ConnectionString, Revision = 7, CreatedUtc = new DateTimeOffset(2026, 1, 2, 3, 4, 5, 678, TimeSpan.FromHours(-5)).AddTicks(9012), UpdatedUtc = new DateTimeOffset(2026, 8, 7, 9, 10, 11, 121, TimeSpan.FromHours(5.5)).AddTicks(3456), CreatedBy = "alice", UpdatedBy = "bob", }; StoredSecret nulls = Row("api/token", 0xB2); StoredSecret empties = Row("svc/empties", 0xC3) with { Description = string.Empty, CreatedBy = string.Empty, UpdatedBy = string.Empty, }; StoredSecret tombstone = Row("retired/legacy", 0xD4) with { IsDeleted = true, DeletedUtc = new DateTimeOffset(2026, 8, 7, 9, 30, 0, TimeSpan.FromHours(9)), Revision = 3, UpdatedBy = "carol", }; await SeedHubAsync(populated, nulls, empties, tombstone); SecretsHubTestClient client = await ConnectAsync(); int pulled = await client.Sweeper.SweepAsync(CancellationToken.None); Assert.Equal(4, pulled); foreach (StoredSecret expected in new[] { populated, nulls, empties, tombstone }) { StoredSecret hubRow = await RequireAsync(_hubStore.Store, expected.Name); StoredSecret localRow = await RequireAsync(_localStore.Store, expected.Name); AssertIdentical(hubRow, localRow); } } [Fact] public async Task Convergence_evicts_the_resolver_cache_for_exactly_the_rows_it_pulled() { await SeedHubAsync(Row("db/central", 0xA1), Row("api/token", 0xB2)); await _localStore.Store.ApplyReplicatedAsync(Row("only/local", 0x11), CancellationToken.None); SecretsHubTestClient client = await ConnectAsync(); await client.Sweeper.SweepAsync(CancellationToken.None); Assert.Equal( ["api/token", "db/central"], client.Invalidated.Select(name => name.Value).Order(StringComparer.Ordinal)); } [Fact] public async Task A_converged_follower_pulls_nothing_on_the_next_sweep() { await SeedHubAsync(Row("db/central", 0xA1)); SecretsHubTestClient client = await ConnectAsync(); await client.Sweeper.SweepAsync(CancellationToken.None); Assert.Equal(0, await client.Sweeper.SweepAsync(CancellationToken.None)); } // ---- 2. delete propagation -------------------------------------------------------------- [Fact] public async Task A_hub_side_tombstone_replaces_a_live_local_row() { StoredSecret live = Row("api/token", 0xB2) with { Revision = 1, CreatedUtc = DateTimeOffset.UtcNow.AddDays(-1), UpdatedUtc = DateTimeOffset.UtcNow.AddDays(-1), }; await SeedHubAsync(live); await _localStore.Store.ApplyReplicatedAsync(live, CancellationToken.None); SecretsHubTestClient client = await ConnectAsync(); // Nothing to do yet: both sides hold the identical live row. Assert.Equal(0, await client.Sweeper.SweepAsync(CancellationToken.None)); // A real delete on the hub — higher revision, newer timestamp, tombstone flag set. Assert.True(await _hubStore.Store.DeleteAsync(live.Name, "admin", CancellationToken.None)); Assert.Equal(1, await client.Sweeper.SweepAsync(CancellationToken.None)); StoredSecret hubRow = await RequireAsync(_hubStore.Store, live.Name); StoredSecret localRow = await RequireAsync(_localStore.Store, live.Name); Assert.True(localRow.IsDeleted); Assert.NotNull(localRow.DeletedUtc); AssertIdentical(hubRow, localRow); } // ---- 3. hub outage at boot, then recovery ----------------------------------------------- [Fact] public async Task A_hub_that_is_down_at_boot_leaves_the_last_known_good_store_intact_and_converges_on_recovery() { StoredSecret lastKnownGood = Row("db/central", 0x77) with { Revision = 1, CreatedUtc = DateTimeOffset.UtcNow.AddDays(-2), UpdatedUtc = DateTimeOffset.UtcNow.AddDays(-2), }; await _localStore.Store.ApplyReplicatedAsync(lastKnownGood, CancellationToken.None); StoredSecret newer = lastKnownGood with { Ciphertext = [0x88], Revision = 2, UpdatedUtc = DateTimeOffset.UtcNow, }; await SeedHubAsync(newer); // The hub exists, but this node cannot reach it: the transport is disconnected before the // node boots, which is the ordinary "central is down while a site restarts" case. var transport = new SwitchableHttpHandler(); SecretsHubTestServer server = await StartHubAsync(); SecretsHubTestClient client = Track( SecretsHubTestClient.Create(_localStore.Store, transport, ConfiguredToken)); await client.Sweeper.StartAsync(CancellationToken.None); await WaitForAsync( () => client.Logs.WarningsFrom().Count > 0, "the startup sweep never reported the outage"); // The node did not crash, and it is still serving exactly what it had. Assert.NotNull(client.Sweeper.ExecuteTask); Assert.False(client.Sweeper.ExecuteTask!.IsFaulted); AssertIdentical(lastKnownGood, await RequireAsync(_localStore.Store, lastKnownGood.Name)); // Central comes back. transport.ConnectTo(server.CreateHandler()); Assert.Equal(1, await client.Sweeper.SweepAsync(CancellationToken.None)); AssertIdentical( await RequireAsync(_hubStore.Store, newer.Name), await RequireAsync(_localStore.Store, newer.Name)); await client.Sweeper.StopAsync(CancellationToken.None); Assert.False(client.Sweeper.ExecuteTask!.IsFaulted); } [Fact] public async Task An_outage_costs_exactly_one_warning_and_no_row_ever_disappears() { // The failure mode this guards is not "the sweep failed" — it is a hub that answers "I have // nothing" being indistinguishable from a hub that does not answer, which would empty the // follower's store. A sweep against a dead transport must move nothing at all. StoredSecret held = Row("api/token", 0x55); await _localStore.Store.ApplyReplicatedAsync(held, CancellationToken.None); var transport = new SwitchableHttpHandler(); SecretsHubTestClient client = Track( SecretsHubTestClient.Create(_localStore.Store, transport, ConfiguredToken)); await client.Sweeper.StartAsync(CancellationToken.None); await WaitForAsync( () => client.Logs.WarningsFrom().Count > 0, "the startup sweep never reported the outage"); await client.Sweeper.StopAsync(CancellationToken.None); Assert.Single(client.Logs.WarningsFrom()); AssertIdentical(held, await RequireAsync(_localStore.Store, held.Name)); } [Fact] public async Task A_transport_failure_reaches_the_raw_sweep_as_a_fault_rather_than_an_empty_answer() { // The containment above is only meaningful because the underlying call really does fail. var transport = new SwitchableHttpHandler(); SecretsHubTestClient client = Track( SecretsHubTestClient.Create( _localStore.Store, transport, ConfiguredToken, syncOnStartup: false)); await Assert.ThrowsAsync( () => client.Sweeper.SweepAsync(CancellationToken.None)); } // ---- 4. authentication, fail-closed, over the wire --------------------------------------- [Fact] public async Task A_wrong_bearer_token_is_refused() { await SeedHubAsync(Row("db/central", 0xA1)); SecretsHubTestServer server = await StartHubAsync(); SecretsHubTestClient client = Track( SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), WrongToken)); RpcException error = await Assert.ThrowsAsync( () => client.Hub.GetManifestAsync(CancellationToken.None)); Assert.Equal(StatusCode.Unauthenticated, error.StatusCode); Assert.Null(await _localStore.Store.GetAsync(new SecretName("db/central"), CancellationToken.None)); } [Fact] public async Task A_call_with_no_credential_at_all_is_refused_in_exactly_the_same_words() { // Uniform denial, asserted where it matters: across the wire, between two DIFFERENT failure // modes. A caller that could tell "no token" from "wrong token" would have a free oracle. SecretsHubTestServer server = await StartHubAsync(); using GrpcChannel channel = ChannelOver(server.CreateHandler()); var bare = new SecretsHub.SecretsHubClient(channel); RpcException noHeader = await Assert.ThrowsAsync( async () => await bare.GetManifestAsync(new GetManifestRequest())); SecretsHubTestClient wrong = Track( SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), WrongToken)); RpcException wrongToken = await Assert.ThrowsAsync( () => wrong.Hub.GetManifestAsync(CancellationToken.None)); Assert.Equal(StatusCode.Unauthenticated, noHeader.StatusCode); Assert.Equal(wrongToken.StatusCode, noHeader.StatusCode); Assert.Equal(wrongToken.Status.Detail, noHeader.Status.Detail); } [Fact] public async Task A_hub_with_no_token_configured_refuses_even_a_well_formed_credential() { SecretsHubTestServer server = Track( await SecretsHubTestServer.StartWithNoTokenConfiguredAsync(_hubStore.Store)); SecretsHubTestClient client = Track( SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken)); RpcException error = await Assert.ThrowsAsync( () => client.Hub.GetManifestAsync(CancellationToken.None)); Assert.Equal(StatusCode.Unauthenticated, error.StatusCode); } [Fact] public async Task An_unconfigured_hub_warns_once_however_many_callers_it_refuses() { // Once per process, not once per call — and the interceptor can only honour that if the // container hands gRPC the singleton it registered. That wiring is exactly what a unit test // constructing the interceptor by hand cannot check. SecretsHubTestServer server = Track( await SecretsHubTestServer.StartWithNoTokenConfiguredAsync(_hubStore.Store)); SecretsHubTestClient client = Track( SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken)); for (int attempt = 0; attempt < 3; attempt++) { await Assert.ThrowsAsync( () => client.Hub.GetManifestAsync(CancellationToken.None)); } // Scoped to the CONFIGURATION warning, because the interceptor deliberately emits a second // kind on this path: a rate-limited summary of the denials themselves. Both are warnings // from the same category, so an unfiltered Single would now be counting two signals as one. IReadOnlyList warnings = server.Logs.WarningsFrom(); CapturedLog warning = Assert.Single( warnings, w => w.Message.Contains("no bearer token configured", StringComparison.Ordinal)); Assert.Contains("Secrets:GrpcHub:BearerToken", warning.Message, StringComparison.Ordinal); // And the denial summary is likewise once, not once per refused caller — the point of the // rate limit, checked here through the real registration rather than a hand-built instance. Assert.Single( warnings, w => w.Message.Contains("The gRPC secrets hub denied", StringComparison.Ordinal)); } [Fact] public async Task No_log_line_on_either_side_carries_credential_material() { await SeedHubAsync(Row("db/central", 0xA1)); SecretsHubTestServer server = await StartHubAsync(); SecretsHubTestClient accepted = Track( SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken)); await accepted.Sweeper.SweepAsync(CancellationToken.None); SecretsHubTestClient refused = Track( SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), WrongToken)); await Assert.ThrowsAsync( () => refused.Hub.GetManifestAsync(CancellationToken.None)); IEnumerable everything = server.Logs.Entries .Concat(accepted.Logs.Entries) .Concat(refused.Logs.Entries); // Captured at Trace on both hosts, so this covers the most verbose setting a deployment // could switch on — the configured token, the presented one, and the header that carried it. Assert.NotEmpty(everything); foreach (CapturedLog entry in everything) { Assert.DoesNotContain(ConfiguredToken, entry.AllText, StringComparison.OrdinalIgnoreCase); Assert.DoesNotContain(WrongToken, entry.AllText, StringComparison.OrdinalIgnoreCase); } } // ---- 5. the request cap, over the wire --------------------------------------------------- [Fact] public async Task A_fetch_over_the_configured_cap_is_refused_as_resource_exhausted() { await SeedHubAsync(Row("db/central", 0xA1), Row("api/token", 0xB2), Row("svc/empties", 0xC3)); SecretsHubTestServer server = await StartHubAsync(maxNamesPerRequest: 2); SecretsHubTestClient client = Track( SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken)); RpcException error = await Assert.ThrowsAsync( () => client.Hub.GetManyAsync( [new SecretName("db/central"), new SecretName("api/token"), new SecretName("svc/empties")], CancellationToken.None)); Assert.Equal(StatusCode.ResourceExhausted, error.StatusCode); } [Fact] public async Task A_fetch_exactly_at_the_cap_is_served() { await SeedHubAsync(Row("db/central", 0xA1), Row("api/token", 0xB2)); SecretsHubTestServer server = await StartHubAsync(maxNamesPerRequest: 2); SecretsHubTestClient client = Track( SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken)); IReadOnlyList rows = await client.Hub.GetManyAsync( [new SecretName("db/central"), new SecretName("api/token")], CancellationToken.None); Assert.Equal(2, rows.Count); } // ---- 6. transport faults stay distinguishable from parse faults -------------------------- [Fact] public async Task An_RpcException_from_the_call_invoker_escapes_the_hub_client_unchanged() { // The client catches PARSE failures per item and drops them; it must never catch a transport // failure. If it did, an unreachable hub would be indistinguishable from an empty one, and // the sweep would have no way to tell "central said nothing" from "central has nothing". var thrown = new RpcException(new Status(StatusCode.Unavailable, "hub unreachable")); var invoker = new FakeSecretsHubCallInvoker { FaultWith = () => thrown }; GrpcSecretsHubClient client = ClientOver(invoker); RpcException fromManifest = await Assert.ThrowsAsync( () => client.GetManifestAsync(CancellationToken.None)); RpcException fromFetch = await Assert.ThrowsAsync( () => client.GetManyAsync([new SecretName("db/central")], CancellationToken.None)); Assert.Same(thrown, fromManifest); Assert.Same(thrown, fromFetch); } // ---- helpers ----------------------------------------------------------------------------- private static GrpcSecretsHubClient ClientOver(CallInvoker invoker) => new( new SecretsHub.SecretsHubClient(invoker), Options.Create(new SecretsGrpcHubClientOptions { Endpoint = "http://central.test", BearerToken = "unused-by-the-invoker", }), new RecordingLogger()); private static GrpcChannel ChannelOver(HttpMessageHandler handler) => GrpcChannel.ForAddress("http://central.test", new GrpcChannelOptions { HttpHandler = handler }); private async Task StartHubAsync(int? maxNamesPerRequest = null) => Track(await SecretsHubTestServer.StartAsync( _hubStore.Store, ConfiguredToken, maxNamesPerRequest)); private async Task ConnectAsync() { SecretsHubTestServer server = await StartHubAsync(); return Track(SecretsHubTestClient.Create( _localStore.Store, server.CreateHandler(), ConfiguredToken, syncOnStartup: false)); } private T Track(T disposable) where T : IAsyncDisposable { _disposables.Add(disposable); return disposable; } // Seeded through ApplyReplicatedAsync rather than UpsertAsync: the upsert path stamps its own // timestamps and revision, which would erase the exact values these tests are about. private async Task SeedHubAsync(params StoredSecret[] rows) { foreach (StoredSecret row in rows) { await _hubStore.Store.ApplyReplicatedAsync(row, CancellationToken.None); } } private static async Task RequireAsync(ISecretStore store, SecretName name) { StoredSecret? row = await store.GetAsync(name, CancellationToken.None); Assert.True(row is not null, $"expected the store to hold '{name.Value}'"); return row!; } private static async Task WaitForAsync(Func condition, string because) { var elapsed = Stopwatch.StartNew(); while (!condition() && elapsed.Elapsed < Patience) { await Task.Delay(10); } Assert.True(condition(), because); } /// /// Field-by-field equality of two rows — every byte array, every timestamp, every nullable. /// /// /// Written out rather than delegated to record equality on purpose: /// holds byte[], which records compare by REFERENCE, so Assert.Equal(a, b) on the /// whole row would pass only for the same instance and silently prove nothing about a row that /// crossed a wire. The byte-identical claim in the live gate is exactly this assertion. /// private static void AssertIdentical(StoredSecret expected, StoredSecret actual) { Assert.Equal(expected.Name.Value, actual.Name.Value); Assert.Equal(expected.Description, actual.Description); Assert.Equal(expected.ContentType, actual.ContentType); Assert.Equal(expected.Ciphertext, actual.Ciphertext); Assert.Equal(expected.Nonce, actual.Nonce); Assert.Equal(expected.Tag, actual.Tag); Assert.Equal(expected.WrappedDek, actual.WrappedDek); Assert.Equal(expected.WrapNonce, actual.WrapNonce); Assert.Equal(expected.WrapTag, actual.WrapTag); Assert.Equal(expected.KekId, actual.KekId); Assert.Equal(expected.Revision, actual.Revision); Assert.Equal(expected.IsDeleted, actual.IsDeleted); Assert.Equal(expected.DeletedUtc, actual.DeletedUtc); Assert.Equal(expected.CreatedUtc, actual.CreatedUtc); Assert.Equal(expected.UpdatedUtc, actual.UpdatedUtc); Assert.Equal(expected.CreatedBy, actual.CreatedBy); Assert.Equal(expected.UpdatedBy, actual.UpdatedBy); // Offset, not just instant: DateTimeOffset equality is instant-only, so a wire form that // normalized every timestamp to UTC would satisfy the assertions above while quietly // discarding the offset the store persisted. Assert.Equal(expected.CreatedUtc.Offset, actual.CreatedUtc.Offset); Assert.Equal(expected.UpdatedUtc.Offset, actual.UpdatedUtc.Offset); Assert.Equal(expected.DeletedUtc?.Offset, actual.DeletedUtc?.Offset); } private static StoredSecret Row(string name, byte marker) => new() { Name = new SecretName(name), ContentType = SecretContentType.Text, Ciphertext = [marker], Nonce = [1, 2], Tag = [3, 4, 5], WrappedDek = [6, 7, 8, 9], WrapNonce = [10], WrapTag = [11, 12], KekId = "sha256:kek-under-test", Revision = 0, CreatedUtc = new DateTimeOffset(2026, 8, 7, 1, 2, 3, TimeSpan.Zero), UpdatedUtc = new DateTimeOffset(2026, 8, 7, 1, 2, 3, TimeSpan.Zero), }; }