fac3c5355c
Secrets-019: the hub channels are created with ThrowOperationCanceledOnCancellation = true, so a call cancelled through its token throws OperationCanceledException. Defensively, the failover reader rethrows RpcException(Cancelled)-with-a-cancelled-token as cancellation instead of dialing every fallback with the same cancelled token, and the sweep treats it as shutdown, not a failure recorded in ISecretsHubSyncStatus. A server-side Cancelled status (token not cancelled) still fails over. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
249 lines
11 KiB
C#
249 lines
11 KiB
C#
using Grpc.Core;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.Secrets.Abstractions;
|
|
using ZB.MOM.WW.Secrets.Replication;
|
|
|
|
namespace ZB.MOM.WW.Secrets.Replicator.Grpc;
|
|
|
|
/// <summary>
|
|
/// Keeps a site node's local secret store converged with the central-hosted hub by running a
|
|
/// <b>pull-only</b> anti-entropy sweep on an interval.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// This is the SQL-Server package's sync service with one half removed, and the removal is the
|
|
/// design. In this topology every update originates at central and a site node is a strict follower,
|
|
/// so there is no push set to compute and no local write to propagate outward. The wire contract has
|
|
/// no write RPC to push with either — the narrow contract IS the enforcement, and this service is
|
|
/// kept honest to match it so the invariant reads the same from both ends.
|
|
/// </para>
|
|
/// <para>
|
|
/// The local store is left completely untouched as a write target for ordinary local use: nothing
|
|
/// here decorates it, so local reads and any local writes behave exactly as they did before
|
|
/// replication was switched on. This sweep is the only writer that carries hub state into it.
|
|
/// </para>
|
|
/// <para>
|
|
/// The sweep never throws out of <see cref="ExecuteAsync"/>: a hub outage must degrade the node to
|
|
/// "serving its last-known-good local secrets" rather than crash the host. Failures are logged once
|
|
/// and retried on the next tick.
|
|
/// </para>
|
|
/// </remarks>
|
|
internal sealed partial class GrpcSecretSyncService : BackgroundService
|
|
{
|
|
private readonly SecretReplicationReconciler _reconciler;
|
|
private readonly ISecretsHubReader _hub;
|
|
private readonly SecretsGrpcHubClientOptions _options;
|
|
private readonly ILogger<GrpcSecretSyncService> _logger;
|
|
private readonly TimeProvider _timeProvider;
|
|
private readonly SecretsHubSyncStatus _status;
|
|
|
|
// Last time a row actually moved, and last time the in-sync heartbeat was logged — both
|
|
// instance state because they describe THIS service's observed history, not anything the hub
|
|
// or the local store can answer on demand.
|
|
private DateTimeOffset? _lastPullUtc;
|
|
private DateTimeOffset? _lastInSyncHeartbeatUtc;
|
|
|
|
/// <summary>Creates the sync service.</summary>
|
|
/// <param name="local">The node's local store — the only thing this service writes to.</param>
|
|
/// <param name="hub">The central hub, read-only.</param>
|
|
/// <param name="cacheInvalidator">Resolver-cache seam, evicted for every applied row.</param>
|
|
/// <param name="options">Hub-client options.</param>
|
|
/// <param name="logger">Logger for sweep outcomes and failures.</param>
|
|
/// <param name="timeProvider">Clock, injectable so tests need not wait a real interval.</param>
|
|
/// <param name="status">
|
|
/// The sweep-outcome tracker this service updates. Optional so existing callers that construct
|
|
/// this type directly (tests, mainly) keep compiling; a real host resolves the same singleton
|
|
/// <c>AddZbSecretsGrpcHubClient</c> registers for <see cref="ISecretsHubSyncStatus"/>, so the two
|
|
/// stay the same instance.
|
|
/// </param>
|
|
public GrpcSecretSyncService(
|
|
ISecretStore local,
|
|
ISecretsHubReader hub,
|
|
ISecretCacheInvalidator? cacheInvalidator,
|
|
IOptions<SecretsGrpcHubClientOptions> options,
|
|
ILogger<GrpcSecretSyncService> logger,
|
|
TimeProvider? timeProvider = null,
|
|
SecretsHubSyncStatus? status = null)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(options);
|
|
ArgumentNullException.ThrowIfNull(local);
|
|
|
|
_hub = hub ?? throw new ArgumentNullException(nameof(hub));
|
|
_options = options.Value;
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
_timeProvider = timeProvider ?? TimeProvider.System;
|
|
_status = status ?? new SecretsHubSyncStatus();
|
|
_reconciler = new SecretReplicationReconciler(
|
|
local,
|
|
cacheInvalidator,
|
|
onRowFailed: (name, ex) => LogRowDiscarded(ex, name.Value));
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
{
|
|
if (_options.SyncOnStartup)
|
|
{
|
|
await SweepSafelyAsync(stoppingToken).ConfigureAwait(false);
|
|
}
|
|
|
|
using PeriodicTimer timer = new(_options.SyncInterval, _timeProvider);
|
|
|
|
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
|
{
|
|
await SweepSafelyAsync(stoppingToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Runs one pull-only reconcile against the hub. Throws on any failure — an unreachable hub, a
|
|
/// rejected credential, a deadline — so a direct caller sees the raw outcome. The background
|
|
/// loop never calls this directly; it goes through <see cref="SweepAndRecordAsync"/>, which
|
|
/// shares this method's reconcile core but also records the outcome and logs it.
|
|
/// </summary>
|
|
/// <param name="ct">A token to cancel the sweep.</param>
|
|
/// <returns>How many rows were pulled from the hub.</returns>
|
|
internal async Task<int> SweepAsync(CancellationToken ct) =>
|
|
(await SweepCoreAsync(ct).ConfigureAwait(false)).Pulled;
|
|
|
|
/// <summary>
|
|
/// The reconcile core shared by <see cref="SweepAsync"/> and <see cref="SweepAndRecordAsync"/>:
|
|
/// fetches the hub manifest and reconciles the local store against it. Throws on failure —
|
|
/// callers that want failures recorded/logged instead do that around this call, not inside it.
|
|
/// </summary>
|
|
private async Task<(int Pulled, int ManifestCount)> SweepCoreAsync(CancellationToken ct)
|
|
{
|
|
IReadOnlyList<SecretManifestEntry> hubManifest =
|
|
await _hub.GetManifestAsync(ct).ConfigureAwait(false);
|
|
|
|
// The reconciler's pull path is exactly the required algorithm — diff the hub's manifest
|
|
// against the local one, fetch what the hub holds newer, apply last-writer-wins, evict the
|
|
// resolver cache per applied row — and it is documented as "a pull, never a push". Calling
|
|
// it rather than re-deriving the diff here also keeps the batching: a cold-starting node's
|
|
// pull set is central's ENTIRE inventory, and an unchunked GetSecrets would exceed gRPC's
|
|
// 4 MB default message cap and then fail identically on every interval forever.
|
|
int pulled = await _reconciler
|
|
.ReconcileAsync(hubManifest, (names, token) => _hub.GetManyAsync(names, token), ct)
|
|
.ConfigureAwait(false);
|
|
|
|
return (pulled, hubManifest.Count);
|
|
}
|
|
|
|
private Task SweepSafelyAsync(CancellationToken ct) => SweepAndRecordAsync(ct);
|
|
|
|
/// <summary>
|
|
/// Test-only entry point for the same outcome-recording sweep the background loop runs — never
|
|
/// throws, always updates <see cref="ISecretsHubSyncStatus"/>, and logs exactly as a real tick
|
|
/// would. Exists so a test can drive one sweep at a time without standing up the
|
|
/// <see cref="BackgroundService"/> host lifecycle.
|
|
/// </summary>
|
|
/// <param name="ct">A token to cancel the sweep.</param>
|
|
internal Task SweepForTestAsync(CancellationToken ct) => SweepAndRecordAsync(ct);
|
|
|
|
private async Task SweepAndRecordAsync(CancellationToken ct)
|
|
{
|
|
DateTimeOffset now = _timeProvider.GetUtcNow();
|
|
|
|
try
|
|
{
|
|
(int pulled, int manifestCount) = await SweepCoreAsync(ct).ConfigureAwait(false);
|
|
|
|
int priorFailures = _status.RecordSuccess(now);
|
|
|
|
if (priorFailures > 0)
|
|
{
|
|
LogSweepRecovered(priorFailures);
|
|
}
|
|
|
|
if (pulled > 0)
|
|
{
|
|
LogSweepConverged(pulled);
|
|
_lastPullUtc = now;
|
|
}
|
|
else
|
|
{
|
|
LogSweepInSync(manifestCount);
|
|
|
|
if (_lastInSyncHeartbeatUtc is null ||
|
|
now - _lastInSyncHeartbeatUtc.Value >= _options.InSyncLogInterval)
|
|
{
|
|
LogSweepInSyncHeartbeat(manifestCount, _lastPullUtc);
|
|
_lastInSyncHeartbeatUtc = now;
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
|
{
|
|
// Host shutdown — not a failure.
|
|
}
|
|
catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled && ct.IsCancellationRequested)
|
|
{
|
|
// Host shutdown, reported gRPC-style by a channel not created through this library's
|
|
// registration (which sets ThrowOperationCanceledOnCancellation) — also not a failure
|
|
// (Secrets-019).
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Deliberately catch-all. Everything reachable from here — an unreachable hub, a
|
|
// rejected credential, a deadline, a row the hub sent that this node's parse refuses —
|
|
// has the same correct response: keep serving the local store and try again. Letting any
|
|
// of it escape would fault the BackgroundService and take the host down over a transient
|
|
// central outage, which is precisely the availability the pull-only topology exists for.
|
|
_status.RecordFailure(now, ex);
|
|
LogSweepFailed(ex);
|
|
}
|
|
}
|
|
|
|
[LoggerMessage(
|
|
EventId = 1,
|
|
Level = LogLevel.Information,
|
|
Message = "Secret hub sync converged: pulled {Pulled} row(s) from the central hub.")]
|
|
private partial void LogSweepConverged(int pulled);
|
|
|
|
[LoggerMessage(
|
|
EventId = 2,
|
|
Level = LogLevel.Warning,
|
|
Message = "Secret hub sync failed; the node continues serving its local store and will retry on the next interval.")]
|
|
private partial void LogSweepFailed(Exception exception);
|
|
|
|
[LoggerMessage(
|
|
EventId = 3,
|
|
Level = LogLevel.Warning,
|
|
Message = "Discarding replicated secret row {SecretName} that could not be applied locally; the rest of the batch continues.")]
|
|
private partial void LogRowDiscarded(Exception exception, string secretName);
|
|
|
|
/// <summary>
|
|
/// Logged at Debug on every sweep that found nothing to pull — the full-cadence signal, for a
|
|
/// verbose log level, that the loop is alive and agrees with the hub even between the rare
|
|
/// Information-level heartbeats.
|
|
/// </summary>
|
|
[LoggerMessage(
|
|
EventId = 4,
|
|
Level = LogLevel.Debug,
|
|
Message = "Secrets hub sweep found nothing new; {Count} secret(s) already in sync.")]
|
|
private partial void LogSweepInSync(int count);
|
|
|
|
/// <summary>
|
|
/// Logged at Information at most once per <see cref="SecretsGrpcHubClientOptions.InSyncLogInterval"/>
|
|
/// — an in-sync sweep is otherwise Debug-only, so this is the periodic proof at the default log
|
|
/// level that the follower is still converged rather than silently stuck.
|
|
/// </summary>
|
|
[LoggerMessage(
|
|
EventId = 5,
|
|
Level = LogLevel.Information,
|
|
Message = "Secrets hub in sync ({Count} secret(s)); last change pulled {LastPullUtc}")]
|
|
private partial void LogSweepInSyncHeartbeat(int count, DateTimeOffset? lastPullUtc);
|
|
|
|
/// <summary>
|
|
/// Logged at Information the first time a sweep succeeds after one or more failures — the
|
|
/// recovery line that closes out whatever run of <see cref="LogSweepFailed"/> Warnings preceded it.
|
|
/// </summary>
|
|
[LoggerMessage(
|
|
EventId = 6,
|
|
Level = LogLevel.Information,
|
|
Message = "Secrets hub sync recovered after {Failures} failed sweep(s)")]
|
|
private partial void LogSweepRecovered(int failures);
|
|
}
|