using Grpc.Core;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Replication;
namespace ZB.MOM.WW.Secrets.Replicator.Grpc;
///
/// Keeps a site node's local secret store converged with the central-hosted hub by running a
/// pull-only anti-entropy sweep on an interval.
///
///
///
/// This is the SQL-Server package's sync service with one half removed, and the removal is the
/// design. In this topology every update originates at central and a site node is a strict follower,
/// so there is no push set to compute and no local write to propagate outward. The wire contract has
/// no write RPC to push with either — the narrow contract IS the enforcement, and this service is
/// kept honest to match it so the invariant reads the same from both ends.
///
///
/// The local store is left completely untouched as a write target for ordinary local use: nothing
/// here decorates it, so local reads and any local writes behave exactly as they did before
/// replication was switched on. This sweep is the only writer that carries hub state into it.
///
///
/// The sweep never throws out of : a hub outage must degrade the node to
/// "serving its last-known-good local secrets" rather than crash the host. Failures are logged once
/// and retried on the next tick.
///
///
internal sealed partial class GrpcSecretSyncService : BackgroundService
{
private readonly SecretReplicationReconciler _reconciler;
private readonly ISecretsHubReader _hub;
private readonly SecretsGrpcHubClientOptions _options;
private readonly ILogger _logger;
private readonly TimeProvider _timeProvider;
private readonly SecretsHubSyncStatus _status;
// Last time a row actually moved, and last time the in-sync heartbeat was logged — both
// instance state because they describe THIS service's observed history, not anything the hub
// or the local store can answer on demand.
private DateTimeOffset? _lastPullUtc;
private DateTimeOffset? _lastInSyncHeartbeatUtc;
/// Creates the sync service.
/// The node's local store — the only thing this service writes to.
/// The central hub, read-only.
/// Resolver-cache seam, evicted for every applied row.
/// Hub-client options.
/// Logger for sweep outcomes and failures.
/// Clock, injectable so tests need not wait a real interval.
///
/// The sweep-outcome tracker this service updates. Optional so existing callers that construct
/// this type directly (tests, mainly) keep compiling; a real host resolves the same singleton
/// AddZbSecretsGrpcHubClient registers for , so the two
/// stay the same instance.
///
public GrpcSecretSyncService(
ISecretStore local,
ISecretsHubReader hub,
ISecretCacheInvalidator? cacheInvalidator,
IOptions options,
ILogger logger,
TimeProvider? timeProvider = null,
SecretsHubSyncStatus? status = null)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(local);
_hub = hub ?? throw new ArgumentNullException(nameof(hub));
_options = options.Value;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_status = status ?? new SecretsHubSyncStatus();
_reconciler = new SecretReplicationReconciler(
local,
cacheInvalidator,
onRowFailed: (name, ex) => LogRowDiscarded(ex, name.Value));
}
///
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
if (_options.SyncOnStartup)
{
await SweepSafelyAsync(stoppingToken).ConfigureAwait(false);
}
using PeriodicTimer timer = new(_options.SyncInterval, _timeProvider);
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
{
await SweepSafelyAsync(stoppingToken).ConfigureAwait(false);
}
}
///
/// Runs one pull-only reconcile against the hub. Throws on any failure — an unreachable hub, a
/// rejected credential, a deadline — so a direct caller sees the raw outcome. The background
/// loop never calls this directly; it goes through , which
/// shares this method's reconcile core but also records the outcome and logs it.
///
/// A token to cancel the sweep.
/// How many rows were pulled from the hub.
internal async Task SweepAsync(CancellationToken ct) =>
(await SweepCoreAsync(ct).ConfigureAwait(false)).Pulled;
///
/// The reconcile core shared by and :
/// fetches the hub manifest and reconciles the local store against it. Throws on failure —
/// callers that want failures recorded/logged instead do that around this call, not inside it.
///
private async Task<(int Pulled, int ManifestCount)> SweepCoreAsync(CancellationToken ct)
{
IReadOnlyList hubManifest =
await _hub.GetManifestAsync(ct).ConfigureAwait(false);
// The reconciler's pull path is exactly the required algorithm — diff the hub's manifest
// against the local one, fetch what the hub holds newer, apply last-writer-wins, evict the
// resolver cache per applied row — and it is documented as "a pull, never a push". Calling
// it rather than re-deriving the diff here also keeps the batching: a cold-starting node's
// pull set is central's ENTIRE inventory, and an unchunked GetSecrets would exceed gRPC's
// 4 MB default message cap and then fail identically on every interval forever.
int pulled = await _reconciler
.ReconcileAsync(hubManifest, (names, token) => _hub.GetManyAsync(names, token), ct)
.ConfigureAwait(false);
return (pulled, hubManifest.Count);
}
private Task SweepSafelyAsync(CancellationToken ct) => SweepAndRecordAsync(ct);
///
/// Test-only entry point for the same outcome-recording sweep the background loop runs — never
/// throws, always updates , and logs exactly as a real tick
/// would. Exists so a test can drive one sweep at a time without standing up the
/// host lifecycle.
///
/// A token to cancel the sweep.
internal Task SweepForTestAsync(CancellationToken ct) => SweepAndRecordAsync(ct);
private async Task SweepAndRecordAsync(CancellationToken ct)
{
DateTimeOffset now = _timeProvider.GetUtcNow();
try
{
(int pulled, int manifestCount) = await SweepCoreAsync(ct).ConfigureAwait(false);
int priorFailures = _status.RecordSuccess(now);
if (priorFailures > 0)
{
LogSweepRecovered(priorFailures);
}
if (pulled > 0)
{
LogSweepConverged(pulled);
_lastPullUtc = now;
}
else
{
LogSweepInSync(manifestCount);
if (_lastInSyncHeartbeatUtc is null ||
now - _lastInSyncHeartbeatUtc.Value >= _options.InSyncLogInterval)
{
LogSweepInSyncHeartbeat(manifestCount, _lastPullUtc);
_lastInSyncHeartbeatUtc = now;
}
}
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
// Host shutdown — not a failure.
}
catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled && ct.IsCancellationRequested)
{
// Host shutdown, reported gRPC-style by a channel not created through this library's
// registration (which sets ThrowOperationCanceledOnCancellation) — also not a failure
// (Secrets-019).
}
catch (Exception ex)
{
// Deliberately catch-all. Everything reachable from here — an unreachable hub, a
// rejected credential, a deadline, a row the hub sent that this node's parse refuses —
// has the same correct response: keep serving the local store and try again. Letting any
// of it escape would fault the BackgroundService and take the host down over a transient
// central outage, which is precisely the availability the pull-only topology exists for.
_status.RecordFailure(now, ex);
LogSweepFailed(ex);
}
}
[LoggerMessage(
EventId = 1,
Level = LogLevel.Information,
Message = "Secret hub sync converged: pulled {Pulled} row(s) from the central hub.")]
private partial void LogSweepConverged(int pulled);
[LoggerMessage(
EventId = 2,
Level = LogLevel.Warning,
Message = "Secret hub sync failed; the node continues serving its local store and will retry on the next interval.")]
private partial void LogSweepFailed(Exception exception);
[LoggerMessage(
EventId = 3,
Level = LogLevel.Warning,
Message = "Discarding replicated secret row {SecretName} that could not be applied locally; the rest of the batch continues.")]
private partial void LogRowDiscarded(Exception exception, string secretName);
///
/// Logged at Debug on every sweep that found nothing to pull — the full-cadence signal, for a
/// verbose log level, that the loop is alive and agrees with the hub even between the rare
/// Information-level heartbeats.
///
[LoggerMessage(
EventId = 4,
Level = LogLevel.Debug,
Message = "Secrets hub sweep found nothing new; {Count} secret(s) already in sync.")]
private partial void LogSweepInSync(int count);
///
/// Logged at Information at most once per
/// — an in-sync sweep is otherwise Debug-only, so this is the periodic proof at the default log
/// level that the follower is still converged rather than silently stuck.
///
[LoggerMessage(
EventId = 5,
Level = LogLevel.Information,
Message = "Secrets hub in sync ({Count} secret(s)); last change pulled {LastPullUtc}")]
private partial void LogSweepInSyncHeartbeat(int count, DateTimeOffset? lastPullUtc);
///
/// Logged at Information the first time a sweep succeeds after one or more failures — the
/// recovery line that closes out whatever run of Warnings preceded it.
///
[LoggerMessage(
EventId = 6,
Level = LogLevel.Information,
Message = "Secrets hub sync recovered after {Failures} failed sweep(s)")]
private partial void LogSweepRecovered(int failures);
}