using System.Diagnostics; using System.Reflection; using Grpc.Net.Client; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.Secrets.Abstractions; using ZB.MOM.WW.Secrets.Replicator.Grpc.DependencyInjection; using ZB.MOM.WW.Secrets.Replicator.Grpc.V1; using ZB.MOM.WW.Secrets.Sqlite; namespace ZB.MOM.WW.Secrets.Replicator.Grpc.Tests; /// /// Convergence behaviour of the follower's pull-only sweep, exercised against real SQLite stores on /// both sides so the assertions are about actual persisted state rather than mock interactions. /// public sealed class GrpcSecretSyncServiceTests : IDisposable { private readonly List _dbPaths = []; private readonly SqliteSecretStore _local; private readonly SqliteSecretStore _hubStore; private readonly FakeHubReader _hub; private readonly RecordingLogger _logger = new(); private readonly RecordingLogger _clientLogger = new(); private readonly RecordingCacheInvalidator _cache = new(); public GrpcSecretSyncServiceTests() { _local = CreateStore(); _hubStore = CreateStore(); _hub = new FakeHubReader(_hubStore); } private SqliteSecretStore CreateStore() { string path = Path.Combine(Path.GetTempPath(), $"zb-grpc-sync-{Guid.NewGuid():N}.db"); _dbPaths.Add(path); var factory = new SecretsSqliteConnectionFactory(path); new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None).GetAwaiter().GetResult(); return new SqliteSecretStore(factory); } private GrpcSecretSyncService CreateService(TimeSpan? interval = null) => new( _local, _hub, _cache, Options.Create(new SecretsGrpcHubClientOptions { Endpoint = "https://central:5001", BearerToken = "unused-by-the-fake", SyncInterval = interval ?? TimeSpan.FromHours(1), }), _logger); private static StoredSecret Row(string name, byte marker) => new() { Name = new SecretName(name), ContentType = SecretContentType.Text, Ciphertext = [marker], Nonce = [1], Tag = [2], WrappedDek = [3], WrapNonce = [4], WrapTag = [5], KekId = "sha256:test", Revision = 0, CreatedUtc = DateTimeOffset.UtcNow, UpdatedUtc = DateTimeOffset.UtcNow, }; [Fact] public async Task Sweep_pulls_a_secret_the_node_has_never_seen() { await _hubStore.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None); int pulled = await CreateService().SweepAsync(CancellationToken.None); Assert.Equal(1, pulled); StoredSecret? local = await _local.GetAsync(new SecretName("db/password"), CancellationToken.None); Assert.NotNull(local); Assert.Equal(0xAA, local!.Ciphertext[0]); } [Fact] public async Task Sweep_pulls_the_row_byte_identically() { await _hubStore.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None); await _hubStore.UpsertAsync(Row("db/password", 0xBB), CancellationToken.None); StoredSecret hubRow = (await _hubStore.GetAsync(new SecretName("db/password"), CancellationToken.None))!; await CreateService().SweepAsync(CancellationToken.None); StoredSecret localRow = (await _local.GetAsync(new SecretName("db/password"), CancellationToken.None))!; Assert.Equal(hubRow.Ciphertext, localRow.Ciphertext); Assert.Equal(hubRow.WrappedDek, localRow.WrappedDek); Assert.Equal(hubRow.KekId, localRow.KekId); Assert.Equal(hubRow.Revision, localRow.Revision); Assert.Equal(hubRow.UpdatedUtc, localRow.UpdatedUtc); } [Fact] public async Task Sweep_applies_a_hub_tombstone_over_a_live_local_row() { await _hubStore.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None); await CreateService().SweepAsync(CancellationToken.None); await _hubStore.DeleteAsync(new SecretName("api/key"), "admin", CancellationToken.None); int pulled = await CreateService().SweepAsync(CancellationToken.None); Assert.Equal(1, pulled); StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!; Assert.True(localRow.IsDeleted); } [Fact] public async Task Sweep_is_idempotent_so_a_converged_follower_stays_quiet() { await _hubStore.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None); GrpcSecretSyncService service = CreateService(); await service.SweepAsync(CancellationToken.None); int pulled = await service.SweepAsync(CancellationToken.None); Assert.Equal(0, pulled); } [Fact] public async Task Sweep_skips_the_fetch_when_the_hub_has_nothing_newer() { await CreateService().SweepAsync(CancellationToken.None); Assert.Equal(0, _hub.GetManyCallCount); } [Fact] public async Task Sweep_never_sends_a_local_only_secret_hub_ward() { // Pull-only: a row this node holds and the hub does not stays put. In the SQL-Server // topology this would be pushed; here central is the sole origin of every update, and the // contract has no write RPC to push it with. await _local.UpsertAsync(Row("only/local", 0x11), CancellationToken.None); int pulled = await CreateService().SweepAsync(CancellationToken.None); Assert.Equal(0, pulled); Assert.Empty(await _hubStore.GetManifestAsync(CancellationToken.None)); } [Fact] public void The_hub_reader_seam_exposes_no_write_operation() { // The structural half of pull-only. The proto has no write RPC; this pins that the client's // own seam cannot grow one either, so a future edit has to break this test on purpose. string[] members = typeof(ISecretsHubReader) .GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly) .Select(m => m.Name) .Order(StringComparer.Ordinal) .ToArray(); Assert.Equal(["GetManifestAsync", "GetManyAsync"], members); } [Theory] [InlineData("ComputePushSet")] [InlineData("ReadLocalAsync")] [InlineData("ApplyReplicatedAsync")] public void The_client_package_never_names_a_hub_ward_write_path(string forbidden) { // ComputePushSet and ReadLocalAsync are the two reconciler entry points that assemble rows // to move toward a peer; ApplyReplicatedAsync on a hub-side store is how they would land. // Pull-only is an owner-confirmed invariant, so the guard is a source scan rather than a // behavioural test: a future edit has to delete this test on purpose to reintroduce a push. foreach (string file in ClientSourceFiles()) { string source = File.ReadAllText(file); Assert.DoesNotContain( forbidden, StripComments(source), StringComparison.Ordinal); } } // The client half of the package — deliberately excludes the server-side files, which legitimately // read a local store to serve the hub. private static IEnumerable ClientSourceFiles() { string root = PackageSourceRoot(); string[] files = [ Path.Combine(root, "GrpcSecretSyncService.cs"), Path.Combine(root, "GrpcSecretsHubClient.cs"), Path.Combine(root, "Wire", "WireToDomain.cs"), Path.Combine(root, "DependencyInjection", "SecretsGrpcHubClientExtensions.cs"), ]; foreach (string file in files) { Assert.True(File.Exists(file), $"expected client source file is missing: {file}"); } return files; } private static string PackageSourceRoot() { DirectoryInfo? dir = new(AppContext.BaseDirectory); while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "ZB.MOM.WW.Secrets.slnx"))) { dir = dir.Parent; } Assert.NotNull(dir); return Path.Combine(dir!.FullName, "src", "ZB.MOM.WW.Secrets.Replicator.Grpc"); } // Comments explain what the code deliberately does NOT do, and naming the forbidden member is // exactly how that is explained — so the scan looks at code only. private static string StripComments(string source) { IEnumerable lines = source .Split('\n') .Select(line => line.TrimStart()) .Where(line => !line.StartsWith("//", StringComparison.Ordinal) && !line.StartsWith("///", StringComparison.Ordinal) && !line.StartsWith('*')); return string.Join('\n', lines); } [Fact] public async Task Sweep_evicts_the_resolver_cache_for_applied_names_only() { await _hubStore.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None); await _local.UpsertAsync(Row("only/local", 0x11), CancellationToken.None); await CreateService().SweepAsync(CancellationToken.None); Assert.Equal(["db/password"], _cache.Invalidated.Select(n => n.Value)); } [Fact] public async Task A_hub_outage_on_the_manifest_call_propagates_out_of_the_raw_sweep() { // Establishes that the fake really fails, so the next two tests prove containment rather // than an accidentally-healthy hub. _hub.FailManifestWith = () => new InvalidOperationException("hub unreachable"); await Assert.ThrowsAsync( () => CreateService().SweepAsync(CancellationToken.None)); } [Fact] public async Task A_hub_outage_on_the_manifest_call_never_escapes_the_background_loop() { await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None); StoredSecret before = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!; _hub.FailManifestWith = () => new InvalidOperationException("hub unreachable"); await RunOneStartupSweepAsync(() => _hub.ManifestCallCount > 0); StoredSecret after = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!; Assert.Equal(before.Ciphertext, after.Ciphertext); Assert.Equal(before.Revision, after.Revision); Assert.Equal(before.UpdatedUtc, after.UpdatedUtc); Assert.False(after.IsDeleted); Assert.Equal(1, _logger.WarningCount); } [Fact] public async Task A_hub_outage_on_the_fetch_call_never_escapes_the_background_loop() { await _hubStore.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None); _hub.FailFetchWith = () => new InvalidOperationException("hub unreachable mid-fetch"); await RunOneStartupSweepAsync(() => _hub.GetManyCallCount > 0); Assert.Null(await _local.GetAsync(new SecretName("db/password"), CancellationToken.None)); Assert.Equal(1, _logger.WarningCount); } [Fact] public async Task A_quiet_sweep_logs_nothing_at_information() { await RunOneStartupSweepAsync(() => _hub.ManifestCallCount > 0); Assert.Equal(0, _logger.InformationCount); Assert.Equal(0, _logger.WarningCount); } [Fact] public async Task A_sweep_that_moved_rows_reports_how_many_it_pulled() { await _hubStore.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None); await RunOneStartupSweepAsync(() => _hub.GetManyCallCount > 0); Assert.Equal(1, _logger.InformationCount); } [Fact] public async Task Startup_sync_can_be_switched_off() { await _hubStore.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None); var service = new GrpcSecretSyncService( _local, _hub, _cache, Options.Create(new SecretsGrpcHubClientOptions { Endpoint = "https://central:5001", BearerToken = "unused-by-the-fake", SyncInterval = TimeSpan.FromHours(1), SyncOnStartup = false, }), _logger); await service.StartAsync(CancellationToken.None); await service.StopAsync(CancellationToken.None); Assert.Equal(0, _hub.ManifestCallCount); } [Fact] public async Task A_malformed_row_never_poisons_the_batch_it_arrived_in() { // The failure this guards against is not "one row is lost" — it is that the pull set never // changes, so the identical malformed row re-fails the identical sweep every interval and // the node never converges on anything, indistinguishable in the log from a hub outage. SecretRow bad = TestWire.Row("api/key", 0xBB); bad.ContentType = "Kerberos"; var invoker = new FakeSecretsHubCallInvoker { ManifestResponse = new GetManifestResponse { Entries = { TestWire.Entry("db/password"), TestWire.Entry("api/key"), TestWire.Entry("svc/token"), }, }, SecretsResponse = new GetSecretsResponse { Secrets = { TestWire.Row("db/password", 0xAA), bad, TestWire.Row("svc/token", 0xCC) }, }, }; int pulled = await CreateServiceOverTheWire(invoker).SweepAsync(CancellationToken.None); Assert.Equal(2, pulled); Assert.NotNull(await _local.GetAsync(new SecretName("db/password"), CancellationToken.None)); Assert.NotNull(await _local.GetAsync(new SecretName("svc/token"), CancellationToken.None)); Assert.Null(await _local.GetAsync(new SecretName("api/key"), CancellationToken.None)); Assert.Single(_clientLogger.Warnings); } [Fact] public async Task A_malformed_manifest_entry_never_poisons_the_manifest_it_arrived_in() { ManifestEntry bad = TestWire.Entry("api/key"); bad.UpdatedUtc = "not-a-timestamp"; var invoker = new FakeSecretsHubCallInvoker { ManifestResponse = new GetManifestResponse { Entries = { TestWire.Entry("db/password"), bad, TestWire.Entry("svc/token") }, }, SecretsResponse = new GetSecretsResponse { Secrets = { TestWire.Row("db/password", 0xAA), TestWire.Row("svc/token", 0xCC) }, }, }; int pulled = await CreateServiceOverTheWire(invoker).SweepAsync(CancellationToken.None); Assert.Equal(2, pulled); Assert.Equal(["db/password", "svc/token"], invoker.LastSecretsRequest!.Names); Assert.Single(_clientLogger.Warnings); } // A sweep wired to the REAL hub client over a fake CallInvoker, so the parse boundary the other // tests bypass (the FakeHubReader hands over domain types already) is actually exercised. private GrpcSecretSyncService CreateServiceOverTheWire(FakeSecretsHubCallInvoker invoker) { IOptions options = Options.Create(new SecretsGrpcHubClientOptions { Endpoint = "https://central:5001", BearerToken = "hub-token", SyncInterval = TimeSpan.FromHours(1), }); var client = new GrpcSecretsHubClient( new SecretsHub.SecretsHubClient(invoker), options, _clientLogger); return new GrpcSecretSyncService(_local, client, _cache, options, _logger); } [Fact] public void Registration_refuses_a_configuration_with_no_bearer_token() { IConfiguration config = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary { ["Secrets:GrpcHub:Endpoint"] = "https://central:5001", }) .Build(); InvalidOperationException ex = Assert.Throws( () => new ServiceCollection().AddZbSecretsGrpcHubClient(config)); Assert.Contains("Secrets:GrpcHub:BearerToken", ex.Message, StringComparison.Ordinal); } [Fact] public void Registration_does_not_hijack_an_application_s_own_grpc_channel() { // GrpcChannel is a framework type, so an application that already has one registered for its // own dependency must keep it — and this package must still dial the hub, not that endpoint. IConfiguration config = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary { ["Secrets:GrpcHub:Endpoint"] = "https://central:5001", ["Secrets:GrpcHub:BearerToken"] = "token", }) .Build(); using GrpcChannel applicationChannel = GrpcChannel.ForAddress("https://somewhere-else:9999"); ServiceCollection services = new(); services.AddSingleton(applicationChannel); services.AddZbSecretsGrpcHubClient(config); using ServiceProvider provider = services.BuildServiceProvider(); Assert.Same(applicationChannel, provider.GetRequiredService()); Assert.NotSame( applicationChannel, provider.GetRequiredKeyedService( SecretsGrpcHubClientExtensions.ChannelServiceKey)); } [Fact] public void Registration_with_no_fallbacks_resolves_the_plain_single_endpoint_client() { // The zero-fallback registration graph is the pre-0.5.0 one, unchanged: one keyed channel, // one GrpcSecretsHubClient as the reader. Consumers wired before FallbackEndpoints existed // must see an identical container shape. ServiceCollection services = new(); services.AddLogging(); services.AddZbSecretsGrpcHubClient(HubConfig()); using ServiceProvider provider = services.BuildServiceProvider(); Assert.IsType(provider.GetRequiredService()); Assert.NotNull(provider.GetRequiredKeyedService( SecretsGrpcHubClientExtensions.ChannelServiceKey)); Assert.Null(provider.GetKeyedService( $"{SecretsGrpcHubClientExtensions.ChannelServiceKey}:fallback:0")); } [Fact] public void Registration_with_fallbacks_resolves_the_failover_reader_over_one_channel_per_endpoint() { ServiceCollection services = new(); services.AddLogging(); services.AddZbSecretsGrpcHubClient(HubConfig(new Dictionary { ["Secrets:GrpcHub:FallbackEndpoints:0"] = "https://central-b:5001", ["Secrets:GrpcHub:FallbackEndpoints:1"] = "https://central-c:5001", })); using ServiceProvider provider = services.BuildServiceProvider(); Assert.IsType(provider.GetRequiredService()); GrpcChannel primary = provider.GetRequiredKeyedService( SecretsGrpcHubClientExtensions.ChannelServiceKey); GrpcChannel fallback0 = provider.GetRequiredKeyedService( $"{SecretsGrpcHubClientExtensions.ChannelServiceKey}:fallback:0"); GrpcChannel fallback1 = provider.GetRequiredKeyedService( $"{SecretsGrpcHubClientExtensions.ChannelServiceKey}:fallback:1"); // Each channel dials its own endpoint — three distinct singletons, container-owned. Assert.Equal("central-a:5001", primary.Target); Assert.Equal("central-b:5001", fallback0.Target); Assert.Equal("central-c:5001", fallback1.Target); } [Fact] public void Registration_refuses_a_malformed_fallback_endpoint_at_startup() { InvalidOperationException ex = Assert.Throws(() => new ServiceCollection().AddZbSecretsGrpcHubClient(HubConfig(new Dictionary { ["Secrets:GrpcHub:FallbackEndpoints:0"] = "not a uri", }))); Assert.Contains("Secrets:GrpcHub:FallbackEndpoints", ex.Message, StringComparison.Ordinal); } // A minimal valid hub-client configuration, extendable per test. private static IConfiguration HubConfig(Dictionary? extra = null) { var values = new Dictionary { ["Secrets:GrpcHub:Endpoint"] = "https://central-a:5001", ["Secrets:GrpcHub:BearerToken"] = "token", }; foreach ((string key, string? value) in extra ?? []) { values[key] = value; } return new ConfigurationBuilder().AddInMemoryCollection(values).Build(); } // Runs the hosted service through one startup sweep and shuts it down, asserting that nothing // escaped ExecuteAsync. The long sync interval means exactly one sweep happens. private async Task RunOneStartupSweepAsync(Func sweepObserved) { GrpcSecretSyncService service = CreateService(); await service.StartAsync(CancellationToken.None); var deadline = Stopwatch.StartNew(); while (!sweepObserved() && deadline.Elapsed < TimeSpan.FromSeconds(10)) { await Task.Delay(10); } await service.StopAsync(CancellationToken.None); Assert.True(sweepObserved(), "the startup sweep never ran"); Assert.NotNull(service.ExecuteTask); Assert.False(service.ExecuteTask!.IsFaulted); } public void Dispose() { Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); foreach (string db in _dbPaths) { foreach (string path in new[] { db, db + "-wal", db + "-shm" }) { if (File.Exists(path)) { File.Delete(path); } } } } /// /// An backed by a real SQLite store, standing in for the /// central-hosted hub. Backed by a real store rather than a dictionary for the same reason the /// SQL-Server suite is: the behaviour under test is last-writer-wins convergence, and a /// hand-rolled fake would re-implement the very logic the tests exist to check. /// private sealed class FakeHubReader(SqliteSecretStore store) : ISecretsHubReader { public int ManifestCallCount { get; private set; } public int GetManyCallCount { get; private set; } public Func? FailManifestWith { get; set; } public Func? FailFetchWith { get; set; } public Task> GetManifestAsync(CancellationToken ct) { ManifestCallCount++; return FailManifestWith is null ? store.GetManifestAsync(ct) : Task.FromException>(FailManifestWith()); } public async Task> GetManyAsync( IReadOnlyList names, CancellationToken ct) { GetManyCallCount++; if (FailFetchWith is not null) { throw FailFetchWith(); } var rows = new List(names.Count); foreach (SecretName name in names) { StoredSecret? row = await store.GetAsync(name, ct); if (row is not null) { rows.Add(row); } } return rows; } } private sealed class RecordingCacheInvalidator : ISecretCacheInvalidator { public List Invalidated { get; } = []; public void Invalidate(SecretName name) => Invalidated.Add(name); } }