Files
scadaproj/ZB.MOM.WW.Secrets/tests/ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests/SqlServerSecretSyncServiceTests.cs
T
Joseph Doherty 64efb49996 fix(secrets): hub push sweep isolates each row (scadaproj#45)
Secrets-008: SqlServerSecretSyncService.SweepAsync now catches a per-row push failure, logs it
(EventId 4, name only) and continues, like the pull side. One row the hub refuses (a >450-char
name, an over-long explicit KEK id) no longer stops every later name from reaching the hub.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 15:32:28 -05:00

224 lines
8.6 KiB
C#

using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Fakes;
using ZB.MOM.WW.Secrets.Sqlite;
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests;
/// <summary>
/// Convergence behaviour of the bidirectional hub sweep, exercised against real stores on both
/// sides so the assertions are about actual persisted state rather than mock interactions.
/// </summary>
public sealed class SqlServerSecretSyncServiceTests : IDisposable
{
private readonly List<string> _dbPaths = [];
private readonly SqliteSecretStore _local;
private readonly SqliteSecretStore _hubStore;
private readonly SqliteBackedHub _hub;
public SqlServerSecretSyncServiceTests()
{
_local = CreateStore();
_hubStore = CreateStore();
_hub = new SqliteBackedHub(_hubStore);
}
private SqliteSecretStore CreateStore()
{
string path = Path.Combine(Path.GetTempPath(), $"zb-sync-{Guid.NewGuid():N}.db");
_dbPaths.Add(path);
var factory = new SecretsSqliteConnectionFactory(path);
new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None).GetAwaiter().GetResult();
return new SqliteSecretStore(factory);
}
private SqlServerSecretSyncService CreateService() => new(
_local,
_hub,
cacheInvalidator: null,
Options.Create(new SqlServerSecretsOptions { ConnectionString = "Server=unused;" }),
NullLogger<SqlServerSecretSyncService>.Instance);
private static StoredSecret Row(string name, byte marker) => new()
{
Name = new SecretName(name),
ContentType = SecretContentType.Text,
Ciphertext = [marker],
Nonce = [1],
Tag = [2],
WrappedDek = [3],
WrapNonce = [4],
WrapTag = [5],
KekId = "sha256:test",
Revision = 0,
CreatedUtc = DateTimeOffset.UtcNow,
UpdatedUtc = DateTimeOffset.UtcNow,
};
[Fact]
public async Task Sweep_pulls_a_secret_the_node_has_never_seen()
{
await _hubStore.UpsertAsync(Row("db/password", 0xAA), CancellationToken.None);
(int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
Assert.Equal(1, pulled);
Assert.Equal(0, pushed);
StoredSecret? local = await _local.GetAsync(new SecretName("db/password"), CancellationToken.None);
Assert.NotNull(local);
Assert.Equal(0xAA, local!.Ciphertext[0]);
}
[Fact]
public async Task Sweep_pushes_a_local_write_the_hub_never_received()
{
// The case that makes best-effort publishing safe: the publish failed (hub was down), so the
// hub has never heard of this name and can never ask for it. Only a push sweep repairs this.
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
(int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
Assert.Equal(0, pulled);
Assert.Equal(1, pushed);
StoredSecret? hub = await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None);
Assert.NotNull(hub);
Assert.Equal(0xBB, hub!.Ciphertext[0]);
}
[Fact]
public async Task Sweep_isolates_a_row_the_hub_rejects_so_later_names_still_push()
{
// Secrets-008: the push set is recomputed in the same name order every interval, so one
// poison row used to stop every name sorting after it, forever.
await _local.UpsertAsync(Row("a/first", 0x01), CancellationToken.None);
await _local.UpsertAsync(Row("m/poison", 0x02), CancellationToken.None);
await _local.UpsertAsync(Row("z/after", 0x03), CancellationToken.None);
_hub.RejectPushOf = "m/poison";
(_, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
Assert.Equal(2, pushed);
Assert.NotNull(await _hubStore.GetAsync(new SecretName("a/first"), CancellationToken.None));
Assert.NotNull(await _hubStore.GetAsync(new SecretName("z/after"), CancellationToken.None));
Assert.Null(await _hubStore.GetAsync(new SecretName("m/poison"), CancellationToken.None));
}
[Fact]
public async Task Sweep_preserves_the_originating_revision_when_pushing()
{
// Pushed verbatim, not upserted: if the hub stamped its own revision the row would look newer
// than the writer's copy and bounce straight back on the next sweep.
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
await _local.UpsertAsync(Row("api/key", 0xCC), CancellationToken.None);
StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
Assert.Equal(1, localRow.Revision);
await CreateService().SweepAsync(CancellationToken.None);
StoredSecret hubRow = (await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
Assert.Equal(localRow.Revision, hubRow.Revision);
Assert.Equal(localRow.UpdatedUtc, hubRow.UpdatedUtc);
}
[Fact]
public async Task Sweep_converges_both_directions_in_one_pass()
{
await _local.UpsertAsync(Row("only/local", 0x11), CancellationToken.None);
await _hubStore.UpsertAsync(Row("only/hub", 0x22), CancellationToken.None);
(int pulled, int pushed) = await CreateService().SweepAsync(CancellationToken.None);
Assert.Equal(1, pulled);
Assert.Equal(1, pushed);
Assert.NotNull(await _local.GetAsync(new SecretName("only/hub"), CancellationToken.None));
Assert.NotNull(await _hubStore.GetAsync(new SecretName("only/local"), CancellationToken.None));
}
[Fact]
public async Task Sweep_is_idempotent_so_a_converged_pair_stays_quiet()
{
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
SqlServerSecretSyncService service = CreateService();
await service.SweepAsync(CancellationToken.None);
(int pulled, int pushed) = await service.SweepAsync(CancellationToken.None);
// A steady-state cluster must not write on every tick — that would churn the hub forever.
Assert.Equal(0, pulled);
Assert.Equal(0, pushed);
}
[Fact]
public async Task Sweep_propagates_a_delete_as_a_tombstone()
{
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
await _local.DeleteAsync(new SecretName("api/key"), "tester", CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
StoredSecret hubRow = (await _hubStore.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
Assert.True(hubRow.IsDeleted);
}
[Fact]
public async Task Sweep_does_not_resurrect_a_secret_deleted_on_the_hub()
{
await _local.UpsertAsync(Row("api/key", 0xBB), CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
await _hubStore.DeleteAsync(new SecretName("api/key"), "admin", CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
Assert.True(localRow.IsDeleted);
}
[Fact]
public async Task Sweep_skips_the_hub_fetch_when_nothing_is_newer()
{
await CreateService().SweepAsync(CancellationToken.None);
Assert.Equal(0, _hub.GetManyCallCount);
}
[Fact]
public async Task Sweep_lets_the_newer_side_win_a_conflicting_write()
{
await _local.UpsertAsync(Row("api/key", 0x11), CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
// Both sides then write independently; the hub's write is later.
await _local.UpsertAsync(Row("api/key", 0x22), CancellationToken.None);
await Task.Delay(10);
await _hubStore.UpsertAsync(Row("api/key", 0x33), CancellationToken.None);
await CreateService().SweepAsync(CancellationToken.None);
StoredSecret localRow = (await _local.GetAsync(new SecretName("api/key"), CancellationToken.None))!;
Assert.Equal(0x33, localRow.Ciphertext[0]);
}
public void Dispose()
{
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
foreach (string db in _dbPaths)
{
foreach (string path in new[] { db, db + "-wal", db + "-shm" })
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
}
}