using Grpc.Core; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.Secrets.Abstractions; using ZB.MOM.WW.Secrets.Replication; namespace ZB.MOM.WW.Secrets.Replicator.Grpc; /// /// Keeps a site node's local secret store converged with the central-hosted hub by running a /// pull-only anti-entropy sweep on an interval. /// /// /// /// This is the SQL-Server package's sync service with one half removed, and the removal is the /// design. In this topology every update originates at central and a site node is a strict follower, /// so there is no push set to compute and no local write to propagate outward. The wire contract has /// no write RPC to push with either — the narrow contract IS the enforcement, and this service is /// kept honest to match it so the invariant reads the same from both ends. /// /// /// The local store is left completely untouched as a write target for ordinary local use: nothing /// here decorates it, so local reads and any local writes behave exactly as they did before /// replication was switched on. This sweep is the only writer that carries hub state into it. /// /// /// The sweep never throws out of : a hub outage must degrade the node to /// "serving its last-known-good local secrets" rather than crash the host. Failures are logged once /// and retried on the next tick. /// /// internal sealed partial class GrpcSecretSyncService : BackgroundService { private readonly SecretReplicationReconciler _reconciler; private readonly ISecretsHubReader _hub; private readonly SecretsGrpcHubClientOptions _options; private readonly ILogger _logger; private readonly TimeProvider _timeProvider; private readonly SecretsHubSyncStatus _status; // Last time a row actually moved, and last time the in-sync heartbeat was logged — both // instance state because they describe THIS service's observed history, not anything the hub // or the local store can answer on demand. private DateTimeOffset? _lastPullUtc; private DateTimeOffset? _lastInSyncHeartbeatUtc; /// Creates the sync service. /// The node's local store — the only thing this service writes to. /// The central hub, read-only. /// Resolver-cache seam, evicted for every applied row. /// Hub-client options. /// Logger for sweep outcomes and failures. /// Clock, injectable so tests need not wait a real interval. /// /// The sweep-outcome tracker this service updates. Optional so existing callers that construct /// this type directly (tests, mainly) keep compiling; a real host resolves the same singleton /// AddZbSecretsGrpcHubClient registers for , so the two /// stay the same instance. /// public GrpcSecretSyncService( ISecretStore local, ISecretsHubReader hub, ISecretCacheInvalidator? cacheInvalidator, IOptions options, ILogger logger, TimeProvider? timeProvider = null, SecretsHubSyncStatus? status = null) { ArgumentNullException.ThrowIfNull(options); ArgumentNullException.ThrowIfNull(local); _hub = hub ?? throw new ArgumentNullException(nameof(hub)); _options = options.Value; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _timeProvider = timeProvider ?? TimeProvider.System; _status = status ?? new SecretsHubSyncStatus(); _reconciler = new SecretReplicationReconciler( local, cacheInvalidator, onRowFailed: (name, ex) => LogRowDiscarded(ex, name.Value)); } /// protected override async Task ExecuteAsync(CancellationToken stoppingToken) { if (_options.SyncOnStartup) { await SweepSafelyAsync(stoppingToken).ConfigureAwait(false); } using PeriodicTimer timer = new(_options.SyncInterval, _timeProvider); while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)) { await SweepSafelyAsync(stoppingToken).ConfigureAwait(false); } } /// /// Runs one pull-only reconcile against the hub. Throws on any failure — an unreachable hub, a /// rejected credential, a deadline — so a direct caller sees the raw outcome. The background /// loop never calls this directly; it goes through , which /// shares this method's reconcile core but also records the outcome and logs it. /// /// A token to cancel the sweep. /// How many rows were pulled from the hub. internal async Task SweepAsync(CancellationToken ct) => (await SweepCoreAsync(ct).ConfigureAwait(false)).Pulled; /// /// The reconcile core shared by and : /// fetches the hub manifest and reconciles the local store against it. Throws on failure — /// callers that want failures recorded/logged instead do that around this call, not inside it. /// private async Task<(int Pulled, int ManifestCount)> SweepCoreAsync(CancellationToken ct) { IReadOnlyList hubManifest = await _hub.GetManifestAsync(ct).ConfigureAwait(false); // The reconciler's pull path is exactly the required algorithm — diff the hub's manifest // against the local one, fetch what the hub holds newer, apply last-writer-wins, evict the // resolver cache per applied row — and it is documented as "a pull, never a push". Calling // it rather than re-deriving the diff here also keeps the batching: a cold-starting node's // pull set is central's ENTIRE inventory, and an unchunked GetSecrets would exceed gRPC's // 4 MB default message cap and then fail identically on every interval forever. int pulled = await _reconciler .ReconcileAsync(hubManifest, (names, token) => _hub.GetManyAsync(names, token), ct) .ConfigureAwait(false); return (pulled, hubManifest.Count); } private Task SweepSafelyAsync(CancellationToken ct) => SweepAndRecordAsync(ct); /// /// Test-only entry point for the same outcome-recording sweep the background loop runs — never /// throws, always updates , and logs exactly as a real tick /// would. Exists so a test can drive one sweep at a time without standing up the /// host lifecycle. /// /// A token to cancel the sweep. internal Task SweepForTestAsync(CancellationToken ct) => SweepAndRecordAsync(ct); private async Task SweepAndRecordAsync(CancellationToken ct) { DateTimeOffset now = _timeProvider.GetUtcNow(); try { (int pulled, int manifestCount) = await SweepCoreAsync(ct).ConfigureAwait(false); int priorFailures = _status.RecordSuccess(now); if (priorFailures > 0) { LogSweepRecovered(priorFailures); } if (pulled > 0) { LogSweepConverged(pulled); _lastPullUtc = now; } else { LogSweepInSync(manifestCount); if (_lastInSyncHeartbeatUtc is null || now - _lastInSyncHeartbeatUtc.Value >= _options.InSyncLogInterval) { LogSweepInSyncHeartbeat(manifestCount, _lastPullUtc); _lastInSyncHeartbeatUtc = now; } } } catch (OperationCanceledException) when (ct.IsCancellationRequested) { // Host shutdown — not a failure. } catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled && ct.IsCancellationRequested) { // Host shutdown, reported gRPC-style by a channel not created through this library's // registration (which sets ThrowOperationCanceledOnCancellation) — also not a failure // (Secrets-019). } catch (Exception ex) { // Deliberately catch-all. Everything reachable from here — an unreachable hub, a // rejected credential, a deadline, a row the hub sent that this node's parse refuses — // has the same correct response: keep serving the local store and try again. Letting any // of it escape would fault the BackgroundService and take the host down over a transient // central outage, which is precisely the availability the pull-only topology exists for. _status.RecordFailure(now, ex); LogSweepFailed(ex); } } [LoggerMessage( EventId = 1, Level = LogLevel.Information, Message = "Secret hub sync converged: pulled {Pulled} row(s) from the central hub.")] private partial void LogSweepConverged(int pulled); [LoggerMessage( EventId = 2, Level = LogLevel.Warning, Message = "Secret hub sync failed; the node continues serving its local store and will retry on the next interval.")] private partial void LogSweepFailed(Exception exception); [LoggerMessage( EventId = 3, Level = LogLevel.Warning, Message = "Discarding replicated secret row {SecretName} that could not be applied locally; the rest of the batch continues.")] private partial void LogRowDiscarded(Exception exception, string secretName); /// /// Logged at Debug on every sweep that found nothing to pull — the full-cadence signal, for a /// verbose log level, that the loop is alive and agrees with the hub even between the rare /// Information-level heartbeats. /// [LoggerMessage( EventId = 4, Level = LogLevel.Debug, Message = "Secrets hub sweep found nothing new; {Count} secret(s) already in sync.")] private partial void LogSweepInSync(int count); /// /// Logged at Information at most once per /// — an in-sync sweep is otherwise Debug-only, so this is the periodic proof at the default log /// level that the follower is still converged rather than silently stuck. /// [LoggerMessage( EventId = 5, Level = LogLevel.Information, Message = "Secrets hub in sync ({Count} secret(s)); last change pulled {LastPullUtc}")] private partial void LogSweepInSyncHeartbeat(int count, DateTimeOffset? lastPullUtc); /// /// Logged at Information the first time a sweep succeeds after one or more failures — the /// recovery line that closes out whatever run of Warnings preceded it. /// [LoggerMessage( EventId = 6, Level = LogLevel.Information, Message = "Secrets hub sync recovered after {Failures} failed sweep(s)")] private partial void LogSweepRecovered(int failures); }