b392b84f21
GrpcSecretSyncService's background sweep now records outcomes on a new SecretsHubSyncStatus (registered singleton, exposed as public ISecretsHubSyncStatus) and logs three new events on the outcome-recording path: EventId 4 (Debug, every in-sync sweep), EventId 5 (Information, the same outcome throttled to at most once per the new SecretsGrpcHubClientOptions.InSyncLogInterval, default 1h), and EventId 6 (Information, the first sweep to succeed after one or more failures). EventIds 1-3 are unchanged; the raw internal SweepAsync still throws on failure exactly as before. Not yet published to the Gitea feed and ScadaBridge is not re-pinned — that is a separate, later step.
251 lines
11 KiB
C#
251 lines
11 KiB
C#
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.Secrets.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.Secrets.Replicator.Grpc.Tests;
|
|
|
|
/// <summary>
|
|
/// Covers the hub follower's sweep-outcome observability: the Debug-per-sweep / Information-
|
|
/// heartbeat-once-per-interval pair for an in-sync sweep (EventId 4/5), the recovered line the
|
|
/// first success after a run of failures logs (EventId 6), and <see cref="ISecretsHubSyncStatus"/>
|
|
/// tracking the same outcomes.
|
|
/// </summary>
|
|
public sealed class GrpcSecretSyncServiceLoggingTests
|
|
{
|
|
[Fact]
|
|
public async Task InSyncSweep_LogsDebugEveryTime_AndInformationHeartbeatOncePerInterval()
|
|
{
|
|
var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-09-28T00:00:00Z"));
|
|
(GrpcSecretSyncService svc, RecordingListLogger logs, _) = Harness.Create(
|
|
time, inSyncLogInterval: TimeSpan.FromHours(1), manifestUnchanged: true);
|
|
|
|
await svc.SweepForTestAsync(CancellationToken.None);
|
|
time.Advance(TimeSpan.FromMinutes(30));
|
|
await svc.SweepForTestAsync(CancellationToken.None);
|
|
time.Advance(TimeSpan.FromMinutes(31));
|
|
await svc.SweepForTestAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(3, logs.Entries.Count(l => l.EventId.Id == 4 && l.Level == LogLevel.Debug));
|
|
Assert.Equal(2, logs.Entries.Count(l => l.EventId.Id == 5 && l.Level == LogLevel.Information));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task FirstSuccessAfterFailures_LogsRecovered_AndStatusResets()
|
|
{
|
|
var time = new FakeTimeProvider();
|
|
(GrpcSecretSyncService svc, RecordingListLogger logs, ISecretsHubSyncStatus status) =
|
|
Harness.Create(time, failFirst: 2);
|
|
|
|
await svc.SweepForTestAsync(default);
|
|
await svc.SweepForTestAsync(default);
|
|
|
|
Assert.Equal(2, status.ConsecutiveFailures);
|
|
|
|
await svc.SweepForTestAsync(default);
|
|
|
|
Assert.Contains(logs.Entries, l => l.EventId.Id == 6 && l.Message.Contains("after 2 failed"));
|
|
Assert.Equal(0, status.ConsecutiveFailures);
|
|
Assert.NotNull(status.LastSuccessUtc);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_failed_sweep_never_logs_EventId6_and_status_records_the_error()
|
|
{
|
|
var time = new FakeTimeProvider();
|
|
(GrpcSecretSyncService svc, RecordingListLogger logs, ISecretsHubSyncStatus status) =
|
|
Harness.Create(time, failFirst: 1);
|
|
|
|
await svc.SweepForTestAsync(default);
|
|
|
|
Assert.DoesNotContain(logs.Entries, l => l.EventId.Id == 6);
|
|
Assert.Equal(1, status.ConsecutiveFailures);
|
|
Assert.NotNull(status.LastError);
|
|
Assert.Null(status.LastSuccessUtc);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task An_in_sync_sweep_never_logs_the_recovered_line()
|
|
{
|
|
var time = new FakeTimeProvider();
|
|
(GrpcSecretSyncService svc, RecordingListLogger logs, ISecretsHubSyncStatus status) =
|
|
Harness.Create(time, manifestUnchanged: true);
|
|
|
|
await svc.SweepForTestAsync(default);
|
|
await svc.SweepForTestAsync(default);
|
|
|
|
Assert.DoesNotContain(logs.Entries, l => l.EventId.Id == 6);
|
|
Assert.Equal(0, status.ConsecutiveFailures);
|
|
Assert.NotNull(status.LastSweepUtc);
|
|
}
|
|
|
|
// Assembles a GrpcSecretSyncService wired to a controllable fake hub, a no-op local store (the
|
|
// scenarios here never actually apply a row — an empty hub manifest means the reconciler's pull
|
|
// set is always empty), a log-capturing logger, and the internal status tracker the service
|
|
// writes to — so a test can both drive sweeps and observe everything they produced.
|
|
private static class Harness
|
|
{
|
|
public static (GrpcSecretSyncService Service, RecordingListLogger Logs, ISecretsHubSyncStatus Status) Create(
|
|
FakeTimeProvider time,
|
|
TimeSpan? inSyncLogInterval = null,
|
|
bool manifestUnchanged = false,
|
|
int failFirst = 0)
|
|
{
|
|
var logger = new RecordingListLogger();
|
|
var status = new SecretsHubSyncStatus();
|
|
|
|
ISecretsHubReader hub = failFirst > 0
|
|
? new SequencedFailThenEmptyHubReader(failFirst)
|
|
: new EmptyHubReader();
|
|
|
|
IOptions<SecretsGrpcHubClientOptions> options = Options.Create(new SecretsGrpcHubClientOptions
|
|
{
|
|
Endpoint = "https://central:5001",
|
|
BearerToken = "unused-by-the-fake",
|
|
SyncInterval = TimeSpan.FromHours(1),
|
|
InSyncLogInterval = inSyncLogInterval ?? TimeSpan.FromHours(1),
|
|
});
|
|
|
|
// manifestUnchanged has no separate fake to select: both readers above always answer an
|
|
// empty (therefore unchanged, nothing-to-pull) manifest once they are done failing, so
|
|
// the parameter exists to make each call site self-documenting rather than to branch on.
|
|
var svc = new GrpcSecretSyncService(
|
|
new NoOpLocalStore(),
|
|
hub,
|
|
cacheInvalidator: null,
|
|
options,
|
|
logger,
|
|
time,
|
|
status);
|
|
|
|
return (svc, logger, status);
|
|
}
|
|
}
|
|
|
|
/// <summary>A hub with nothing in it — every sweep is in-sync with zero rows to pull.</summary>
|
|
private sealed class EmptyHubReader : ISecretsHubReader
|
|
{
|
|
public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
|
|
Task.FromResult<IReadOnlyList<SecretManifestEntry>>([]);
|
|
|
|
public Task<IReadOnlyList<StoredSecret>> GetManyAsync(
|
|
IReadOnlyList<SecretName> names, CancellationToken ct) =>
|
|
Task.FromResult<IReadOnlyList<StoredSecret>>([]);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Fails the manifest call with a transport-shaped exception for the first
|
|
/// <paramref name="failuresBeforeSuccess"/> calls, then behaves exactly like
|
|
/// <see cref="EmptyHubReader"/> forever after.
|
|
/// </summary>
|
|
private sealed class SequencedFailThenEmptyHubReader(int failuresBeforeSuccess) : ISecretsHubReader
|
|
{
|
|
private int _remaining = failuresBeforeSuccess;
|
|
|
|
public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct)
|
|
{
|
|
if (_remaining > 0)
|
|
{
|
|
_remaining--;
|
|
return Task.FromException<IReadOnlyList<SecretManifestEntry>>(
|
|
new InvalidOperationException("hub unreachable"));
|
|
}
|
|
|
|
return Task.FromResult<IReadOnlyList<SecretManifestEntry>>([]);
|
|
}
|
|
|
|
public Task<IReadOnlyList<StoredSecret>> GetManyAsync(
|
|
IReadOnlyList<SecretName> names, CancellationToken ct) =>
|
|
Task.FromResult<IReadOnlyList<StoredSecret>>([]);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A local store that is never actually exercised by these scenarios (the hub manifest is
|
|
/// always empty, so the reconciler's pull set is always empty and none of these members are
|
|
/// called) beyond <see cref="GetManifestAsync"/>, which the reconciler always calls to compute
|
|
/// the pull set.
|
|
/// </summary>
|
|
private sealed class NoOpLocalStore : ISecretStore
|
|
{
|
|
public Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct) =>
|
|
throw new NotSupportedException("not exercised by the logging test scenarios.");
|
|
|
|
public Task UpsertAsync(StoredSecret row, CancellationToken ct) =>
|
|
throw new NotSupportedException("not exercised by the logging test scenarios.");
|
|
|
|
public Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct) =>
|
|
throw new NotSupportedException("not exercised by the logging test scenarios.");
|
|
|
|
public Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct) =>
|
|
throw new NotSupportedException("not exercised by the logging test scenarios.");
|
|
|
|
public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
|
|
Task.FromResult<IReadOnlyList<SecretManifestEntry>>([]);
|
|
|
|
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) =>
|
|
throw new NotSupportedException("not exercised by the logging test scenarios.");
|
|
|
|
public Task<bool> ApplyRewrapAsync(
|
|
StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
|
|
throw new NotSupportedException("not exercised by the logging test scenarios.");
|
|
}
|
|
}
|
|
|
|
/// <summary>A clock a test advances by hand, so an hourly heartbeat can be tested without waiting an hour.</summary>
|
|
/// <remarks>
|
|
/// Hand-rolled rather than taken from <c>Microsoft.Extensions.TimeProvider.Testing</c>, matching
|
|
/// this project's existing convention (see <c>HubFakeTimeProvider</c> in
|
|
/// <c>SecretsHubAuthInterceptorTests.cs</c>) of not taking that package dependency for one small type.
|
|
/// </remarks>
|
|
internal sealed class FakeTimeProvider : TimeProvider
|
|
{
|
|
private DateTimeOffset _now;
|
|
|
|
/// <summary>Creates a clock starting at the current wall-clock time.</summary>
|
|
public FakeTimeProvider() : this(DateTimeOffset.UtcNow)
|
|
{
|
|
}
|
|
|
|
/// <summary>Creates a clock starting at <paramref name="now"/>.</summary>
|
|
/// <param name="now">The instant the clock starts at.</param>
|
|
public FakeTimeProvider(DateTimeOffset now) => _now = now;
|
|
|
|
/// <inheritdoc/>
|
|
public override DateTimeOffset GetUtcNow() => _now;
|
|
|
|
/// <summary>Moves the clock forward by <paramref name="delta"/>.</summary>
|
|
/// <param name="delta">The amount of time to advance.</param>
|
|
public void Advance(TimeSpan delta) => _now += delta;
|
|
}
|
|
|
|
/// <summary>Captures every log entry's level, event id, and formatted message, across all levels.</summary>
|
|
/// <remarks>
|
|
/// Distinct from the existing <c>RecordingLogger<T></c> in <c>TestDoubles.cs</c>, which only
|
|
/// captures Information/Warning and neither type captures <see cref="EventId"/> — this suite needs
|
|
/// Debug-level entries and event-id assertions that the existing double does not support, and
|
|
/// widening it would risk the tests that already depend on its narrower shape.
|
|
/// </remarks>
|
|
internal sealed class RecordingListLogger : ILogger<GrpcSecretSyncService>
|
|
{
|
|
/// <summary>Every entry logged, in order.</summary>
|
|
public List<(LogLevel Level, EventId EventId, string Message)> Entries { get; } = [];
|
|
|
|
/// <inheritdoc/>
|
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
|
|
|
/// <inheritdoc/>
|
|
public bool IsEnabled(LogLevel logLevel) => true;
|
|
|
|
/// <inheritdoc/>
|
|
public void Log<TState>(
|
|
LogLevel logLevel,
|
|
EventId eventId,
|
|
TState state,
|
|
Exception? exception,
|
|
Func<TState, Exception?, string> formatter)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(formatter);
|
|
|
|
Entries.Add((logLevel, eventId, formatter(state, exception)));
|
|
}
|
|
}
|