Files
lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GalaxyAckIdentityDirectoryTests.cs
T
Joseph Doherty 0702b31029
v2-ci / unit-tests (push) Waiting to run
fix(alarms): refuse a keyed native ack with no Galaxy identity synchronously, from a cached key set (lmxopcua#765)
Item 3 of #765. A keyed (API-key) session whose key had no Galaxy ack identity
got Good and then a revert, because whether the key has an identity lives in
the Secrets store, which must not be read under the SDK lock.

GalaxyAckIdentityDirectory (Driver.Galaxy) keeps a node-wide set of the key ids
that have a galaxy-ack-user secret. It lists this node's Secrets store
(metadata only, nothing decrypted) and is read lock-free.
GalaxyDriver.GetAcknowledgeRefusal (IAlarmAcknowledgePreflight, published
through NativeAlarmAckPreflightView as in #760) now refuses a keyed session
whose key the directory reports Absent, when the driver has no fallback ack
account. A refusal also asks for a refresh.

Fails open: before the first refresh, once the snapshot is older than
StaleAfter, and from a signalled identity-secret change until the next
refresh, the lookup answers Unknown and the ack takes the old Good-then-revert
path.

Refresh: GalaxyAckIdentityRefreshService runs on driver nodes, every
Drivers:Galaxy:AckIdentityCache:RefreshIntervalSeconds (default 30;
StaleAfterSeconds 0 = three intervals; validated at start). It also refreshes
early on secret changes. The Secrets library has no change event, so
SecretsRegistration decorates ISecretCacheInvalidator, which is called by
AdminUI /api-keys, /admin/secrets and the replication appliers. API-key
changes need no event: a revoked key is already denied before the pre-check.

Tests: Galaxy.Tests GalaxyAckIdentityDirectoryTests (16) and Host.Tests
GalaxyAckIdentityCacheWiringTests (9). Seven separate mutations each turn at
least one test red: pre-check ignores the directory, factory drops it, no
invalidator decoration, no staleness check, no change-pending check, no
refresh on refusal, loop never wakes early.
Docs: Galaxy.md, AlarmTracking.md, Configuration.md, .claude/rules/alarms.md.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 16:38:57 -05:00

357 lines
15 KiB
C#

using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
using ZB.MOM.WW.Secrets.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// lmxopcua#765 item 3 — a keyed session whose API key has no Galaxy ack identity is refused synchronously
/// (<c>BadUserAccessDenied</c> via <see cref="ZB.MOM.WW.OtOpcUa.Core.Abstractions.IAlarmAcknowledgePreflight"/>)
/// from a cached, periodically refreshed set read without I/O, and everything the set is not certain about
/// fails open to the Good-then-revert path.
/// </summary>
public sealed class GalaxyAckIdentityDirectoryTests
{
private const string WithIdentity = "0a1b2c3d4e5f6071";
private const string WithoutIdentity = "ffeeddccbbaa0099";
private static readonly TimeSpan Interval = TimeSpan.FromSeconds(30);
private static readonly TimeSpan StaleAfter = TimeSpan.FromSeconds(90);
// ------------------------------------------------------------------ driver pre-check
[Fact]
public async Task Keyed_session_without_an_identity_is_refused_synchronously()
{
var (directory, _, _) = await RefreshedDirectoryAsync();
using var driver = NewDriver(directory);
var refusal = driver.GetAcknowledgeRefusal(WithoutIdentity);
refusal.ShouldNotBeNull();
refusal.ShouldContain(WithoutIdentity);
refusal.ShouldContain("no Galaxy ack identity");
refusal.ShouldContain("drv");
}
[Fact]
public async Task Keyed_session_with_an_identity_proceeds()
{
var (directory, _, _) = await RefreshedDirectoryAsync();
using var driver = NewDriver(directory);
driver.GetAcknowledgeRefusal(WithIdentity).ShouldBeNull();
driver.GetAcknowledgeRefusal(WithIdentity.ToUpperInvariant()).ShouldBeNull("key ids are case-insensitive");
}
[Fact]
public async Task A_fallback_account_lets_a_keyed_session_without_an_identity_through()
{
var (directory, _, _) = await RefreshedDirectoryAsync();
using var driver = NewDriver(directory, Opts() with
{
AlarmAck = new GalaxyAlarmAckOptions("ack-svc", "PLANT", "secret:otopcua/galaxy/ack-fallback"),
});
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldBeNull();
}
[Fact]
public void Without_a_directory_a_keyed_session_is_never_pre_refused()
{
using var driver = NewDriver(directory: null);
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldBeNull();
driver.GetAcknowledgeRefusal(apiKeyId: null).ShouldNotBeNull("the #760 anonymous refusal is unchanged");
}
[Fact]
public async Task A_refusal_asks_for_a_refresh_so_a_retry_sees_an_identity_set_out_of_band()
{
var (directory, store, _) = await RefreshedDirectoryAsync(TimeSpan.FromHours(1));
using var driver = NewDriver(directory);
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var loop = directory.RunAsync(cts.Token);
await WaitForAsync(() => store.ListCalls >= 2, cts.Token); // RunAsync's own initial refresh
// Set by the `secret` CLI: nothing in this process is told.
store.AddIdentity(WithoutIdentity);
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldNotBeNull();
// The refusal kicked a refresh well inside the one-hour interval.
await WaitForAsync(() => driver.GetAcknowledgeRefusal(WithoutIdentity) is null, cts.Token);
await cts.CancelAsync();
await loop;
}
// ------------------------------------------------------------------ fail open
[Fact]
public void Before_the_first_refresh_every_key_is_Unknown_and_proceeds()
{
using var directory = new GalaxyAckIdentityDirectory(new FakeSecretStore(), Interval, StaleAfter);
using var driver = NewDriver(directory);
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldBeNull();
}
[Fact]
public async Task A_stale_snapshot_is_Unknown_and_proceeds()
{
var (directory, store, clock) = await RefreshedDirectoryAsync();
using var driver = NewDriver(directory);
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldNotBeNull("positive control: fresh snapshot refuses");
// The store starts failing: the old snapshot is kept, and ages out.
store.Fail = true;
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeFalse();
clock.Advance(StaleAfter - TimeSpan.FromSeconds(1));
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent);
clock.Advance(TimeSpan.FromSeconds(2));
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
directory.GetPresence(WithIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldBeNull();
}
[Fact]
public async Task A_signalled_identity_change_makes_the_snapshot_Unknown_until_the_next_refresh()
{
var (directory, store, _) = await RefreshedDirectoryAsync();
using var driver = NewDriver(directory);
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldNotBeNull("positive control");
// AdminUI sets the identity: the write invalidates the secret, the directory is told.
store.AddIdentity(WithoutIdentity);
directory.NotifySecretChanged(new SecretName(GalaxyAckIdentitySecrets.UserSecretName(WithoutIdentity)));
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldBeNull("never refuse in the window before the refresh");
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Present);
}
[Fact]
public async Task A_change_signalled_during_a_refresh_leaves_that_snapshot_untrusted()
{
var (directory, store, _) = await RefreshedDirectoryAsync();
store.OnList = () => directory.NotifySecretChanged(
new SecretName(GalaxyAckIdentitySecrets.PasswordSecretName(WithoutIdentity)));
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
}
[Fact]
public async Task An_unrelated_secret_change_does_not_untrust_the_snapshot()
{
var (directory, _, _) = await RefreshedDirectoryAsync();
directory.NotifySecretChanged(new SecretName("otopcua/galaxy/ack-fallback"));
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent);
}
[Fact]
public async Task A_directory_without_a_store_is_always_Unknown()
{
using var directory = new GalaxyAckIdentityDirectory(store: null, Interval, StaleAfter);
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeFalse();
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
}
[Fact]
public async Task A_key_id_the_resolver_could_not_look_up_is_Unknown()
{
var (directory, _, _) = await RefreshedDirectoryAsync();
directory.GetPresence("../etc").ShouldBe(GalaxyAckIdentityPresence.Unknown);
directory.GetPresence(" ").ShouldBe(GalaxyAckIdentityPresence.Unknown);
directory.GetPresence(null).ShouldBe(GalaxyAckIdentityPresence.Unknown);
}
// ------------------------------------------------------------------ refresh
[Fact]
public async Task Refresh_picks_up_added_and_removed_identities()
{
var (directory, store, _) = await RefreshedDirectoryAsync();
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent);
store.AddIdentity(WithoutIdentity);
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent, "a cache, until refreshed");
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Present);
store.RemoveIdentity(WithIdentity, softDelete: true);
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
directory.GetPresence(WithIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent, "a deleted user secret is no identity");
}
[Fact]
public async Task Only_user_secrets_count_as_an_identity()
{
var store = new FakeSecretStore();
store.Add(GalaxyAckIdentitySecrets.PasswordSecretName(WithoutIdentity)); // password without user
store.Add("otopcua/galaxy/ack-fallback");
using var directory = new GalaxyAckIdentityDirectory(store, Interval, StaleAfter);
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent);
}
[Fact]
public async Task The_loop_refreshes_on_its_interval()
{
var store = new FakeSecretStore();
using var directory = new GalaxyAckIdentityDirectory(store, TimeSpan.FromMilliseconds(50), TimeSpan.FromMinutes(1));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var loop = directory.RunAsync(cts.Token);
await WaitForAsync(() => directory.GetPresence(WithIdentity) == GalaxyAckIdentityPresence.Absent, cts.Token);
store.AddIdentity(WithIdentity);
await WaitForAsync(() => directory.GetPresence(WithIdentity) == GalaxyAckIdentityPresence.Present, cts.Token);
await cts.CancelAsync();
await loop; // completes cleanly on cancellation
}
[Fact]
public async Task The_loop_refreshes_early_on_a_signalled_change()
{
var store = new FakeSecretStore();
using var directory = new GalaxyAckIdentityDirectory(store, TimeSpan.FromHours(1), TimeSpan.FromHours(3));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var loop = directory.RunAsync(cts.Token);
await WaitForAsync(() => directory.GetPresence(WithIdentity) == GalaxyAckIdentityPresence.Absent, cts.Token);
store.AddIdentity(WithIdentity);
directory.NotifySecretChanged(new SecretName(GalaxyAckIdentitySecrets.UserSecretName(WithIdentity)));
await WaitForAsync(() => directory.GetPresence(WithIdentity) == GalaxyAckIdentityPresence.Present, cts.Token);
await cts.CancelAsync();
await loop;
}
// ------------------------------------------------------------------ helpers
private static async Task<(GalaxyAckIdentityDirectory Directory, FakeSecretStore Store, ManualClock Clock)>
RefreshedDirectoryAsync(TimeSpan? interval = null)
{
var store = new FakeSecretStore();
store.AddIdentity(WithIdentity);
var clock = new ManualClock();
var directory = new GalaxyAckIdentityDirectory(
store, interval ?? Interval, interval is { } i ? i * 3 : StaleAfter, clock);
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
return (directory, store, clock);
}
private static async Task WaitForAsync(Func<bool> condition, CancellationToken ct)
{
while (!condition())
{
ct.ThrowIfCancellationRequested();
await Task.Delay(10, ct);
}
}
private static GalaxyDriverOptions Opts() => new(
new GalaxyGatewayOptions("https://mxgw.test:5001", "dev:key"),
new GalaxyMxAccessOptions("OtOpcUa-A"),
new GalaxyRepositoryOptions(),
new GalaxyReconnectOptions());
private static GalaxyDriver NewDriver(IGalaxyAckIdentityDirectory? directory, GalaxyDriverOptions? options = null) =>
new("drv", options ?? Opts(), hierarchySource: null, ackIdentityDirectory: directory);
private sealed class ManualClock : TimeProvider
{
private DateTimeOffset _now = new(2026, 11, 9, 12, 0, 0, TimeSpan.Zero);
public void Advance(TimeSpan by) => _now += by;
public override DateTimeOffset GetUtcNow() => _now;
}
/// <summary>Metadata-only store: <see cref="ListAsync"/> is all the directory may call.</summary>
private sealed class FakeSecretStore : ISecretStore
{
private readonly Dictionary<string, bool> _rows = new(StringComparer.Ordinal); // name → deleted
private readonly Lock _lock = new();
private int _listCalls;
public bool Fail { get; set; }
public Action? OnList { get; set; }
public int ListCalls => Volatile.Read(ref _listCalls);
public void Add(string name)
{
lock (_lock) _rows[name] = false;
}
public void AddIdentity(string keyId)
{
Add(GalaxyAckIdentitySecrets.PasswordSecretName(keyId));
Add(GalaxyAckIdentitySecrets.UserSecretName(keyId));
}
public void RemoveIdentity(string keyId, bool softDelete)
{
lock (_lock)
{
foreach (var name in new[]
{
GalaxyAckIdentitySecrets.UserSecretName(keyId), GalaxyAckIdentitySecrets.PasswordSecretName(keyId),
})
{
if (softDelete) _rows[name] = true;
else _rows.Remove(name);
}
}
}
public Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct)
{
Interlocked.Increment(ref _listCalls);
OnList?.Invoke();
if (Fail) throw new InvalidOperationException("store unavailable");
lock (_lock)
{
// Deleted rows are returned regardless, to prove the directory filters them itself.
IReadOnlyList<SecretMetadata> rows = _rows.Select(r => new SecretMetadata
{
Name = new SecretName(r.Key),
ContentType = SecretContentType.Text,
KekId = "k",
Revision = 1,
IsDeleted = r.Value,
CreatedUtc = DateTimeOffset.UnixEpoch,
UpdatedUtc = DateTimeOffset.UnixEpoch,
}).ToList();
return Task.FromResult(rows);
}
}
public Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct) => throw Unexpected();
public Task UpsertAsync(StoredSecret row, CancellationToken ct) => throw Unexpected();
public Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct) => throw Unexpected();
public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) => throw Unexpected();
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) => throw Unexpected();
public Task<bool> ApplyRewrapAsync(StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
throw Unexpected();
// The directory must read metadata only — never a row (that would decrypt-adjacent I/O per key).
private static InvalidOperationException Unexpected() => new("the directory may only list metadata");
}
}