using System.Diagnostics;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Replicator.Grpc.V1;
namespace ZB.MOM.WW.Secrets.Replicator.Grpc.Tests;
///
/// The gRPC secrets hub end to end: a central node hosting the real service through its own DI and
/// interceptor, a site node running the real client and sweep through its own DI, and a real gRPC
/// call between them.
///
///
///
/// Everything the unit suites necessarily substitute is real here — the registration extensions, the
/// generated client and service, the fail-closed interceptor as the container actually activates it,
/// protobuf encoding on both sides, and two real SQLite stores. What that buys is the class of defect
/// no unit test can reach: pieces that are each correct and are never connected to one another.
///
///
/// No credential in this file is a real one. The two token values are fabricated for these
/// tests, and one of the tests exists to assert that neither ever reaches a log sink on either side.
///
///
public sealed class SecretsHubIntegrationTests : IAsyncLifetime
{
private const string ConfiguredToken = "central-hub-credential-7f3a";
private const string WrongToken = "impostor-credential-91cd";
private static readonly TimeSpan Patience = TimeSpan.FromSeconds(10);
private readonly TempSecretStore _hubStore = new("hub");
private readonly TempSecretStore _localStore = new("site");
private readonly List _disposables = [];
///
public Task InitializeAsync() => Task.CompletedTask;
///
public async Task DisposeAsync()
{
foreach (IAsyncDisposable disposable in _disposables)
{
await disposable.DisposeAsync();
}
_localStore.Dispose();
_hubStore.Dispose();
}
// ---- 1. end-to-end convergence ----------------------------------------------------------
[Fact]
public async Task One_sweep_carries_every_row_across_the_wire_byte_identically()
{
// The four rows between them cover every shape the wire has to keep distinguishable: a fully
// populated row with non-UTC offsets on both timestamps, a row whose every nullable field is
// null, a row whose nullable strings are deliberately EMPTY rather than absent, and a
// tombstone. Empty-versus-null is the one a plain proto3 string would silently collapse.
StoredSecret populated = Row("db/central", 0xA1) with
{
Description = "central connection string",
ContentType = SecretContentType.ConnectionString,
Revision = 7,
CreatedUtc = new DateTimeOffset(2026, 1, 2, 3, 4, 5, 678, TimeSpan.FromHours(-5)).AddTicks(9012),
UpdatedUtc = new DateTimeOffset(2026, 8, 7, 9, 10, 11, 121, TimeSpan.FromHours(5.5)).AddTicks(3456),
CreatedBy = "alice",
UpdatedBy = "bob",
};
StoredSecret nulls = Row("api/token", 0xB2);
StoredSecret empties = Row("svc/empties", 0xC3) with
{
Description = string.Empty,
CreatedBy = string.Empty,
UpdatedBy = string.Empty,
};
StoredSecret tombstone = Row("retired/legacy", 0xD4) with
{
IsDeleted = true,
DeletedUtc = new DateTimeOffset(2026, 8, 7, 9, 30, 0, TimeSpan.FromHours(9)),
Revision = 3,
UpdatedBy = "carol",
};
await SeedHubAsync(populated, nulls, empties, tombstone);
SecretsHubTestClient client = await ConnectAsync();
int pulled = await client.Sweeper.SweepAsync(CancellationToken.None);
Assert.Equal(4, pulled);
foreach (StoredSecret expected in new[] { populated, nulls, empties, tombstone })
{
StoredSecret hubRow = await RequireAsync(_hubStore.Store, expected.Name);
StoredSecret localRow = await RequireAsync(_localStore.Store, expected.Name);
AssertIdentical(hubRow, localRow);
}
}
[Fact]
public async Task Convergence_evicts_the_resolver_cache_for_exactly_the_rows_it_pulled()
{
await SeedHubAsync(Row("db/central", 0xA1), Row("api/token", 0xB2));
await _localStore.Store.ApplyReplicatedAsync(Row("only/local", 0x11), CancellationToken.None);
SecretsHubTestClient client = await ConnectAsync();
await client.Sweeper.SweepAsync(CancellationToken.None);
Assert.Equal(
["api/token", "db/central"],
client.Invalidated.Select(name => name.Value).Order(StringComparer.Ordinal));
}
[Fact]
public async Task A_converged_follower_pulls_nothing_on_the_next_sweep()
{
await SeedHubAsync(Row("db/central", 0xA1));
SecretsHubTestClient client = await ConnectAsync();
await client.Sweeper.SweepAsync(CancellationToken.None);
Assert.Equal(0, await client.Sweeper.SweepAsync(CancellationToken.None));
}
// ---- 2. delete propagation --------------------------------------------------------------
[Fact]
public async Task A_hub_side_tombstone_replaces_a_live_local_row()
{
StoredSecret live = Row("api/token", 0xB2) with
{
Revision = 1,
CreatedUtc = DateTimeOffset.UtcNow.AddDays(-1),
UpdatedUtc = DateTimeOffset.UtcNow.AddDays(-1),
};
await SeedHubAsync(live);
await _localStore.Store.ApplyReplicatedAsync(live, CancellationToken.None);
SecretsHubTestClient client = await ConnectAsync();
// Nothing to do yet: both sides hold the identical live row.
Assert.Equal(0, await client.Sweeper.SweepAsync(CancellationToken.None));
// A real delete on the hub — higher revision, newer timestamp, tombstone flag set.
Assert.True(await _hubStore.Store.DeleteAsync(live.Name, "admin", CancellationToken.None));
Assert.Equal(1, await client.Sweeper.SweepAsync(CancellationToken.None));
StoredSecret hubRow = await RequireAsync(_hubStore.Store, live.Name);
StoredSecret localRow = await RequireAsync(_localStore.Store, live.Name);
Assert.True(localRow.IsDeleted);
Assert.NotNull(localRow.DeletedUtc);
AssertIdentical(hubRow, localRow);
}
// ---- 3. hub outage at boot, then recovery -----------------------------------------------
[Fact]
public async Task A_hub_that_is_down_at_boot_leaves_the_last_known_good_store_intact_and_converges_on_recovery()
{
StoredSecret lastKnownGood = Row("db/central", 0x77) with
{
Revision = 1,
CreatedUtc = DateTimeOffset.UtcNow.AddDays(-2),
UpdatedUtc = DateTimeOffset.UtcNow.AddDays(-2),
};
await _localStore.Store.ApplyReplicatedAsync(lastKnownGood, CancellationToken.None);
StoredSecret newer = lastKnownGood with
{
Ciphertext = [0x88],
Revision = 2,
UpdatedUtc = DateTimeOffset.UtcNow,
};
await SeedHubAsync(newer);
// The hub exists, but this node cannot reach it: the transport is disconnected before the
// node boots, which is the ordinary "central is down while a site restarts" case.
var transport = new SwitchableHttpHandler();
SecretsHubTestServer server = await StartHubAsync();
SecretsHubTestClient client = Track(
SecretsHubTestClient.Create(_localStore.Store, transport, ConfiguredToken));
await client.Sweeper.StartAsync(CancellationToken.None);
await WaitForAsync(
() => client.Logs.WarningsFrom().Count > 0,
"the startup sweep never reported the outage");
// The node did not crash, and it is still serving exactly what it had.
Assert.NotNull(client.Sweeper.ExecuteTask);
Assert.False(client.Sweeper.ExecuteTask!.IsFaulted);
AssertIdentical(lastKnownGood, await RequireAsync(_localStore.Store, lastKnownGood.Name));
// Central comes back.
transport.ConnectTo(server.CreateHandler());
Assert.Equal(1, await client.Sweeper.SweepAsync(CancellationToken.None));
AssertIdentical(
await RequireAsync(_hubStore.Store, newer.Name),
await RequireAsync(_localStore.Store, newer.Name));
await client.Sweeper.StopAsync(CancellationToken.None);
Assert.False(client.Sweeper.ExecuteTask!.IsFaulted);
}
[Fact]
public async Task An_outage_costs_exactly_one_warning_and_no_row_ever_disappears()
{
// The failure mode this guards is not "the sweep failed" — it is a hub that answers "I have
// nothing" being indistinguishable from a hub that does not answer, which would empty the
// follower's store. A sweep against a dead transport must move nothing at all.
StoredSecret held = Row("api/token", 0x55);
await _localStore.Store.ApplyReplicatedAsync(held, CancellationToken.None);
var transport = new SwitchableHttpHandler();
SecretsHubTestClient client = Track(
SecretsHubTestClient.Create(_localStore.Store, transport, ConfiguredToken));
await client.Sweeper.StartAsync(CancellationToken.None);
await WaitForAsync(
() => client.Logs.WarningsFrom().Count > 0,
"the startup sweep never reported the outage");
await client.Sweeper.StopAsync(CancellationToken.None);
Assert.Single(client.Logs.WarningsFrom());
AssertIdentical(held, await RequireAsync(_localStore.Store, held.Name));
}
[Fact]
public async Task A_transport_failure_reaches_the_raw_sweep_as_a_fault_rather_than_an_empty_answer()
{
// The containment above is only meaningful because the underlying call really does fail.
var transport = new SwitchableHttpHandler();
SecretsHubTestClient client = Track(
SecretsHubTestClient.Create(
_localStore.Store, transport, ConfiguredToken, syncOnStartup: false));
await Assert.ThrowsAsync(
() => client.Sweeper.SweepAsync(CancellationToken.None));
}
// ---- 4. authentication, fail-closed, over the wire ---------------------------------------
[Fact]
public async Task A_wrong_bearer_token_is_refused()
{
await SeedHubAsync(Row("db/central", 0xA1));
SecretsHubTestServer server = await StartHubAsync();
SecretsHubTestClient client = Track(
SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), WrongToken));
RpcException error = await Assert.ThrowsAsync(
() => client.Hub.GetManifestAsync(CancellationToken.None));
Assert.Equal(StatusCode.Unauthenticated, error.StatusCode);
Assert.Null(await _localStore.Store.GetAsync(new SecretName("db/central"), CancellationToken.None));
}
[Fact]
public async Task A_call_with_no_credential_at_all_is_refused_in_exactly_the_same_words()
{
// Uniform denial, asserted where it matters: across the wire, between two DIFFERENT failure
// modes. A caller that could tell "no token" from "wrong token" would have a free oracle.
SecretsHubTestServer server = await StartHubAsync();
using GrpcChannel channel = ChannelOver(server.CreateHandler());
var bare = new SecretsHub.SecretsHubClient(channel);
RpcException noHeader = await Assert.ThrowsAsync(
async () => await bare.GetManifestAsync(new GetManifestRequest()));
SecretsHubTestClient wrong = Track(
SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), WrongToken));
RpcException wrongToken = await Assert.ThrowsAsync(
() => wrong.Hub.GetManifestAsync(CancellationToken.None));
Assert.Equal(StatusCode.Unauthenticated, noHeader.StatusCode);
Assert.Equal(wrongToken.StatusCode, noHeader.StatusCode);
Assert.Equal(wrongToken.Status.Detail, noHeader.Status.Detail);
}
[Fact]
public async Task A_hub_with_no_token_configured_refuses_even_a_well_formed_credential()
{
SecretsHubTestServer server = Track(
await SecretsHubTestServer.StartWithNoTokenConfiguredAsync(_hubStore.Store));
SecretsHubTestClient client = Track(
SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken));
RpcException error = await Assert.ThrowsAsync(
() => client.Hub.GetManifestAsync(CancellationToken.None));
Assert.Equal(StatusCode.Unauthenticated, error.StatusCode);
}
[Fact]
public async Task An_unconfigured_hub_warns_once_however_many_callers_it_refuses()
{
// Once per process, not once per call — and the interceptor can only honour that if the
// container hands gRPC the singleton it registered. That wiring is exactly what a unit test
// constructing the interceptor by hand cannot check.
SecretsHubTestServer server = Track(
await SecretsHubTestServer.StartWithNoTokenConfiguredAsync(_hubStore.Store));
SecretsHubTestClient client = Track(
SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken));
for (int attempt = 0; attempt < 3; attempt++)
{
await Assert.ThrowsAsync(
() => client.Hub.GetManifestAsync(CancellationToken.None));
}
// Scoped to the CONFIGURATION warning, because the interceptor deliberately emits a second
// kind on this path: a rate-limited summary of the denials themselves. Both are warnings
// from the same category, so an unfiltered Single would now be counting two signals as one.
IReadOnlyList warnings = server.Logs.WarningsFrom();
CapturedLog warning = Assert.Single(
warnings, w => w.Message.Contains("no bearer token configured", StringComparison.Ordinal));
Assert.Contains("Secrets:GrpcHub:BearerToken", warning.Message, StringComparison.Ordinal);
// And the denial summary is likewise once, not once per refused caller — the point of the
// rate limit, checked here through the real registration rather than a hand-built instance.
Assert.Single(
warnings, w => w.Message.Contains("The gRPC secrets hub denied", StringComparison.Ordinal));
}
[Fact]
public async Task No_log_line_on_either_side_carries_credential_material()
{
await SeedHubAsync(Row("db/central", 0xA1));
SecretsHubTestServer server = await StartHubAsync();
SecretsHubTestClient accepted = Track(
SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken));
await accepted.Sweeper.SweepAsync(CancellationToken.None);
SecretsHubTestClient refused = Track(
SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), WrongToken));
await Assert.ThrowsAsync(
() => refused.Hub.GetManifestAsync(CancellationToken.None));
IEnumerable everything = server.Logs.Entries
.Concat(accepted.Logs.Entries)
.Concat(refused.Logs.Entries);
// Captured at Trace on both hosts, so this covers the most verbose setting a deployment
// could switch on — the configured token, the presented one, and the header that carried it.
Assert.NotEmpty(everything);
foreach (CapturedLog entry in everything)
{
Assert.DoesNotContain(ConfiguredToken, entry.AllText, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain(WrongToken, entry.AllText, StringComparison.OrdinalIgnoreCase);
}
}
// ---- 5. the request cap, over the wire ---------------------------------------------------
[Fact]
public async Task A_fetch_over_the_configured_cap_is_refused_as_resource_exhausted()
{
await SeedHubAsync(Row("db/central", 0xA1), Row("api/token", 0xB2), Row("svc/empties", 0xC3));
SecretsHubTestServer server = await StartHubAsync(maxNamesPerRequest: 2);
SecretsHubTestClient client = Track(
SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken));
RpcException error = await Assert.ThrowsAsync(
() => client.Hub.GetManyAsync(
[new SecretName("db/central"), new SecretName("api/token"), new SecretName("svc/empties")],
CancellationToken.None));
Assert.Equal(StatusCode.ResourceExhausted, error.StatusCode);
}
[Fact]
public async Task A_fetch_exactly_at_the_cap_is_served()
{
await SeedHubAsync(Row("db/central", 0xA1), Row("api/token", 0xB2));
SecretsHubTestServer server = await StartHubAsync(maxNamesPerRequest: 2);
SecretsHubTestClient client = Track(
SecretsHubTestClient.Create(_localStore.Store, server.CreateHandler(), ConfiguredToken));
IReadOnlyList rows = await client.Hub.GetManyAsync(
[new SecretName("db/central"), new SecretName("api/token")], CancellationToken.None);
Assert.Equal(2, rows.Count);
}
// ---- 6. transport faults stay distinguishable from parse faults --------------------------
[Fact]
public async Task An_RpcException_from_the_call_invoker_escapes_the_hub_client_unchanged()
{
// The client catches PARSE failures per item and drops them; it must never catch a transport
// failure. If it did, an unreachable hub would be indistinguishable from an empty one, and
// the sweep would have no way to tell "central said nothing" from "central has nothing".
var thrown = new RpcException(new Status(StatusCode.Unavailable, "hub unreachable"));
var invoker = new FakeSecretsHubCallInvoker { FaultWith = () => thrown };
GrpcSecretsHubClient client = ClientOver(invoker);
RpcException fromManifest = await Assert.ThrowsAsync(
() => client.GetManifestAsync(CancellationToken.None));
RpcException fromFetch = await Assert.ThrowsAsync(
() => client.GetManyAsync([new SecretName("db/central")], CancellationToken.None));
Assert.Same(thrown, fromManifest);
Assert.Same(thrown, fromFetch);
}
// ---- helpers -----------------------------------------------------------------------------
private static GrpcSecretsHubClient ClientOver(CallInvoker invoker) => new(
new SecretsHub.SecretsHubClient(invoker),
Options.Create(new SecretsGrpcHubClientOptions
{
Endpoint = "http://central.test",
BearerToken = "unused-by-the-invoker",
}),
new RecordingLogger());
private static GrpcChannel ChannelOver(HttpMessageHandler handler) =>
GrpcChannel.ForAddress("http://central.test", new GrpcChannelOptions { HttpHandler = handler });
private async Task StartHubAsync(int? maxNamesPerRequest = null) =>
Track(await SecretsHubTestServer.StartAsync(
_hubStore.Store, ConfiguredToken, maxNamesPerRequest));
private async Task ConnectAsync()
{
SecretsHubTestServer server = await StartHubAsync();
return Track(SecretsHubTestClient.Create(
_localStore.Store, server.CreateHandler(), ConfiguredToken, syncOnStartup: false));
}
private T Track(T disposable) where T : IAsyncDisposable
{
_disposables.Add(disposable);
return disposable;
}
// Seeded through ApplyReplicatedAsync rather than UpsertAsync: the upsert path stamps its own
// timestamps and revision, which would erase the exact values these tests are about.
private async Task SeedHubAsync(params StoredSecret[] rows)
{
foreach (StoredSecret row in rows)
{
await _hubStore.Store.ApplyReplicatedAsync(row, CancellationToken.None);
}
}
private static async Task RequireAsync(ISecretStore store, SecretName name)
{
StoredSecret? row = await store.GetAsync(name, CancellationToken.None);
Assert.True(row is not null, $"expected the store to hold '{name.Value}'");
return row!;
}
private static async Task WaitForAsync(Func condition, string because)
{
var elapsed = Stopwatch.StartNew();
while (!condition() && elapsed.Elapsed < Patience)
{
await Task.Delay(10);
}
Assert.True(condition(), because);
}
///
/// Field-by-field equality of two rows — every byte array, every timestamp, every nullable.
///
///
/// Written out rather than delegated to record equality on purpose:
/// holds byte[], which records compare by REFERENCE, so Assert.Equal(a, b) on the
/// whole row would pass only for the same instance and silently prove nothing about a row that
/// crossed a wire. The byte-identical claim in the live gate is exactly this assertion.
///
private static void AssertIdentical(StoredSecret expected, StoredSecret actual)
{
Assert.Equal(expected.Name.Value, actual.Name.Value);
Assert.Equal(expected.Description, actual.Description);
Assert.Equal(expected.ContentType, actual.ContentType);
Assert.Equal(expected.Ciphertext, actual.Ciphertext);
Assert.Equal(expected.Nonce, actual.Nonce);
Assert.Equal(expected.Tag, actual.Tag);
Assert.Equal(expected.WrappedDek, actual.WrappedDek);
Assert.Equal(expected.WrapNonce, actual.WrapNonce);
Assert.Equal(expected.WrapTag, actual.WrapTag);
Assert.Equal(expected.KekId, actual.KekId);
Assert.Equal(expected.Revision, actual.Revision);
Assert.Equal(expected.IsDeleted, actual.IsDeleted);
Assert.Equal(expected.DeletedUtc, actual.DeletedUtc);
Assert.Equal(expected.CreatedUtc, actual.CreatedUtc);
Assert.Equal(expected.UpdatedUtc, actual.UpdatedUtc);
Assert.Equal(expected.CreatedBy, actual.CreatedBy);
Assert.Equal(expected.UpdatedBy, actual.UpdatedBy);
// Offset, not just instant: DateTimeOffset equality is instant-only, so a wire form that
// normalized every timestamp to UTC would satisfy the assertions above while quietly
// discarding the offset the store persisted.
Assert.Equal(expected.CreatedUtc.Offset, actual.CreatedUtc.Offset);
Assert.Equal(expected.UpdatedUtc.Offset, actual.UpdatedUtc.Offset);
Assert.Equal(expected.DeletedUtc?.Offset, actual.DeletedUtc?.Offset);
}
private static StoredSecret Row(string name, byte marker) => new()
{
Name = new SecretName(name),
ContentType = SecretContentType.Text,
Ciphertext = [marker],
Nonce = [1, 2],
Tag = [3, 4, 5],
WrappedDek = [6, 7, 8, 9],
WrapNonce = [10],
WrapTag = [11, 12],
KekId = "sha256:kek-under-test",
Revision = 0,
CreatedUtc = new DateTimeOffset(2026, 8, 7, 1, 2, 3, TimeSpan.Zero),
UpdatedUtc = new DateTimeOffset(2026, 8, 7, 1, 2, 3, TimeSpan.Zero),
};
}