0702b31029
v2-ci / unit-tests (push) Waiting to run
Item 3 of #765. A keyed (API-key) session whose key had no Galaxy ack identity got Good and then a revert, because whether the key has an identity lives in the Secrets store, which must not be read under the SDK lock. GalaxyAckIdentityDirectory (Driver.Galaxy) keeps a node-wide set of the key ids that have a galaxy-ack-user secret. It lists this node's Secrets store (metadata only, nothing decrypted) and is read lock-free. GalaxyDriver.GetAcknowledgeRefusal (IAlarmAcknowledgePreflight, published through NativeAlarmAckPreflightView as in #760) now refuses a keyed session whose key the directory reports Absent, when the driver has no fallback ack account. A refusal also asks for a refresh. Fails open: before the first refresh, once the snapshot is older than StaleAfter, and from a signalled identity-secret change until the next refresh, the lookup answers Unknown and the ack takes the old Good-then-revert path. Refresh: GalaxyAckIdentityRefreshService runs on driver nodes, every Drivers:Galaxy:AckIdentityCache:RefreshIntervalSeconds (default 30; StaleAfterSeconds 0 = three intervals; validated at start). It also refreshes early on secret changes. The Secrets library has no change event, so SecretsRegistration decorates ISecretCacheInvalidator, which is called by AdminUI /api-keys, /admin/secrets and the replication appliers. API-key changes need no event: a revoked key is already denied before the pre-check. Tests: Galaxy.Tests GalaxyAckIdentityDirectoryTests (16) and Host.Tests GalaxyAckIdentityCacheWiringTests (9). Seven separate mutations each turn at least one test red: pre-check ignores the directory, factory drops it, no invalidator decoration, no staleness check, no change-pending check, no refresh on refusal, loop never wakes early. Docs: Galaxy.md, AlarmTracking.md, Configuration.md, .claude/rules/alarms.md. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
357 lines
15 KiB
C#
357 lines
15 KiB
C#
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
|
using ZB.MOM.WW.Secrets.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
|
|
|
/// <summary>
|
|
/// lmxopcua#765 item 3 — a keyed session whose API key has no Galaxy ack identity is refused synchronously
|
|
/// (<c>BadUserAccessDenied</c> via <see cref="ZB.MOM.WW.OtOpcUa.Core.Abstractions.IAlarmAcknowledgePreflight"/>)
|
|
/// from a cached, periodically refreshed set read without I/O, and everything the set is not certain about
|
|
/// fails open to the Good-then-revert path.
|
|
/// </summary>
|
|
public sealed class GalaxyAckIdentityDirectoryTests
|
|
{
|
|
private const string WithIdentity = "0a1b2c3d4e5f6071";
|
|
private const string WithoutIdentity = "ffeeddccbbaa0099";
|
|
private static readonly TimeSpan Interval = TimeSpan.FromSeconds(30);
|
|
private static readonly TimeSpan StaleAfter = TimeSpan.FromSeconds(90);
|
|
|
|
// ------------------------------------------------------------------ driver pre-check
|
|
|
|
[Fact]
|
|
public async Task Keyed_session_without_an_identity_is_refused_synchronously()
|
|
{
|
|
var (directory, _, _) = await RefreshedDirectoryAsync();
|
|
using var driver = NewDriver(directory);
|
|
|
|
var refusal = driver.GetAcknowledgeRefusal(WithoutIdentity);
|
|
|
|
refusal.ShouldNotBeNull();
|
|
refusal.ShouldContain(WithoutIdentity);
|
|
refusal.ShouldContain("no Galaxy ack identity");
|
|
refusal.ShouldContain("drv");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Keyed_session_with_an_identity_proceeds()
|
|
{
|
|
var (directory, _, _) = await RefreshedDirectoryAsync();
|
|
using var driver = NewDriver(directory);
|
|
|
|
driver.GetAcknowledgeRefusal(WithIdentity).ShouldBeNull();
|
|
driver.GetAcknowledgeRefusal(WithIdentity.ToUpperInvariant()).ShouldBeNull("key ids are case-insensitive");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_fallback_account_lets_a_keyed_session_without_an_identity_through()
|
|
{
|
|
var (directory, _, _) = await RefreshedDirectoryAsync();
|
|
using var driver = NewDriver(directory, Opts() with
|
|
{
|
|
AlarmAck = new GalaxyAlarmAckOptions("ack-svc", "PLANT", "secret:otopcua/galaxy/ack-fallback"),
|
|
});
|
|
|
|
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Without_a_directory_a_keyed_session_is_never_pre_refused()
|
|
{
|
|
using var driver = NewDriver(directory: null);
|
|
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldBeNull();
|
|
driver.GetAcknowledgeRefusal(apiKeyId: null).ShouldNotBeNull("the #760 anonymous refusal is unchanged");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_refusal_asks_for_a_refresh_so_a_retry_sees_an_identity_set_out_of_band()
|
|
{
|
|
var (directory, store, _) = await RefreshedDirectoryAsync(TimeSpan.FromHours(1));
|
|
using var driver = NewDriver(directory);
|
|
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
|
var loop = directory.RunAsync(cts.Token);
|
|
await WaitForAsync(() => store.ListCalls >= 2, cts.Token); // RunAsync's own initial refresh
|
|
|
|
// Set by the `secret` CLI: nothing in this process is told.
|
|
store.AddIdentity(WithoutIdentity);
|
|
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldNotBeNull();
|
|
|
|
// The refusal kicked a refresh well inside the one-hour interval.
|
|
await WaitForAsync(() => driver.GetAcknowledgeRefusal(WithoutIdentity) is null, cts.Token);
|
|
|
|
await cts.CancelAsync();
|
|
await loop;
|
|
}
|
|
|
|
// ------------------------------------------------------------------ fail open
|
|
|
|
[Fact]
|
|
public void Before_the_first_refresh_every_key_is_Unknown_and_proceeds()
|
|
{
|
|
using var directory = new GalaxyAckIdentityDirectory(new FakeSecretStore(), Interval, StaleAfter);
|
|
using var driver = NewDriver(directory);
|
|
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
|
|
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_stale_snapshot_is_Unknown_and_proceeds()
|
|
{
|
|
var (directory, store, clock) = await RefreshedDirectoryAsync();
|
|
using var driver = NewDriver(directory);
|
|
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldNotBeNull("positive control: fresh snapshot refuses");
|
|
|
|
// The store starts failing: the old snapshot is kept, and ages out.
|
|
store.Fail = true;
|
|
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeFalse();
|
|
clock.Advance(StaleAfter - TimeSpan.FromSeconds(1));
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent);
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(2));
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
|
|
directory.GetPresence(WithIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
|
|
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_signalled_identity_change_makes_the_snapshot_Unknown_until_the_next_refresh()
|
|
{
|
|
var (directory, store, _) = await RefreshedDirectoryAsync();
|
|
using var driver = NewDriver(directory);
|
|
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldNotBeNull("positive control");
|
|
|
|
// AdminUI sets the identity: the write invalidates the secret, the directory is told.
|
|
store.AddIdentity(WithoutIdentity);
|
|
directory.NotifySecretChanged(new SecretName(GalaxyAckIdentitySecrets.UserSecretName(WithoutIdentity)));
|
|
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
|
|
driver.GetAcknowledgeRefusal(WithoutIdentity).ShouldBeNull("never refuse in the window before the refresh");
|
|
|
|
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Present);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_change_signalled_during_a_refresh_leaves_that_snapshot_untrusted()
|
|
{
|
|
var (directory, store, _) = await RefreshedDirectoryAsync();
|
|
store.OnList = () => directory.NotifySecretChanged(
|
|
new SecretName(GalaxyAckIdentitySecrets.PasswordSecretName(WithoutIdentity)));
|
|
|
|
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
|
|
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task An_unrelated_secret_change_does_not_untrust_the_snapshot()
|
|
{
|
|
var (directory, _, _) = await RefreshedDirectoryAsync();
|
|
directory.NotifySecretChanged(new SecretName("otopcua/galaxy/ack-fallback"));
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_directory_without_a_store_is_always_Unknown()
|
|
{
|
|
using var directory = new GalaxyAckIdentityDirectory(store: null, Interval, StaleAfter);
|
|
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeFalse();
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Unknown);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_key_id_the_resolver_could_not_look_up_is_Unknown()
|
|
{
|
|
var (directory, _, _) = await RefreshedDirectoryAsync();
|
|
directory.GetPresence("../etc").ShouldBe(GalaxyAckIdentityPresence.Unknown);
|
|
directory.GetPresence(" ").ShouldBe(GalaxyAckIdentityPresence.Unknown);
|
|
directory.GetPresence(null).ShouldBe(GalaxyAckIdentityPresence.Unknown);
|
|
}
|
|
|
|
// ------------------------------------------------------------------ refresh
|
|
|
|
[Fact]
|
|
public async Task Refresh_picks_up_added_and_removed_identities()
|
|
{
|
|
var (directory, store, _) = await RefreshedDirectoryAsync();
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent);
|
|
|
|
store.AddIdentity(WithoutIdentity);
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent, "a cache, until refreshed");
|
|
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Present);
|
|
|
|
store.RemoveIdentity(WithIdentity, softDelete: true);
|
|
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
|
|
directory.GetPresence(WithIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent, "a deleted user secret is no identity");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Only_user_secrets_count_as_an_identity()
|
|
{
|
|
var store = new FakeSecretStore();
|
|
store.Add(GalaxyAckIdentitySecrets.PasswordSecretName(WithoutIdentity)); // password without user
|
|
store.Add("otopcua/galaxy/ack-fallback");
|
|
using var directory = new GalaxyAckIdentityDirectory(store, Interval, StaleAfter);
|
|
|
|
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
|
|
|
|
directory.GetPresence(WithoutIdentity).ShouldBe(GalaxyAckIdentityPresence.Absent);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task The_loop_refreshes_on_its_interval()
|
|
{
|
|
var store = new FakeSecretStore();
|
|
using var directory = new GalaxyAckIdentityDirectory(store, TimeSpan.FromMilliseconds(50), TimeSpan.FromMinutes(1));
|
|
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
|
var loop = directory.RunAsync(cts.Token);
|
|
|
|
await WaitForAsync(() => directory.GetPresence(WithIdentity) == GalaxyAckIdentityPresence.Absent, cts.Token);
|
|
store.AddIdentity(WithIdentity);
|
|
await WaitForAsync(() => directory.GetPresence(WithIdentity) == GalaxyAckIdentityPresence.Present, cts.Token);
|
|
|
|
await cts.CancelAsync();
|
|
await loop; // completes cleanly on cancellation
|
|
}
|
|
|
|
[Fact]
|
|
public async Task The_loop_refreshes_early_on_a_signalled_change()
|
|
{
|
|
var store = new FakeSecretStore();
|
|
using var directory = new GalaxyAckIdentityDirectory(store, TimeSpan.FromHours(1), TimeSpan.FromHours(3));
|
|
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
|
var loop = directory.RunAsync(cts.Token);
|
|
await WaitForAsync(() => directory.GetPresence(WithIdentity) == GalaxyAckIdentityPresence.Absent, cts.Token);
|
|
|
|
store.AddIdentity(WithIdentity);
|
|
directory.NotifySecretChanged(new SecretName(GalaxyAckIdentitySecrets.UserSecretName(WithIdentity)));
|
|
|
|
await WaitForAsync(() => directory.GetPresence(WithIdentity) == GalaxyAckIdentityPresence.Present, cts.Token);
|
|
await cts.CancelAsync();
|
|
await loop;
|
|
}
|
|
|
|
// ------------------------------------------------------------------ helpers
|
|
|
|
private static async Task<(GalaxyAckIdentityDirectory Directory, FakeSecretStore Store, ManualClock Clock)>
|
|
RefreshedDirectoryAsync(TimeSpan? interval = null)
|
|
{
|
|
var store = new FakeSecretStore();
|
|
store.AddIdentity(WithIdentity);
|
|
var clock = new ManualClock();
|
|
var directory = new GalaxyAckIdentityDirectory(
|
|
store, interval ?? Interval, interval is { } i ? i * 3 : StaleAfter, clock);
|
|
(await directory.RefreshAsync(CancellationToken.None)).ShouldBeTrue();
|
|
return (directory, store, clock);
|
|
}
|
|
|
|
private static async Task WaitForAsync(Func<bool> condition, CancellationToken ct)
|
|
{
|
|
while (!condition())
|
|
{
|
|
ct.ThrowIfCancellationRequested();
|
|
await Task.Delay(10, ct);
|
|
}
|
|
}
|
|
|
|
private static GalaxyDriverOptions Opts() => new(
|
|
new GalaxyGatewayOptions("https://mxgw.test:5001", "dev:key"),
|
|
new GalaxyMxAccessOptions("OtOpcUa-A"),
|
|
new GalaxyRepositoryOptions(),
|
|
new GalaxyReconnectOptions());
|
|
|
|
private static GalaxyDriver NewDriver(IGalaxyAckIdentityDirectory? directory, GalaxyDriverOptions? options = null) =>
|
|
new("drv", options ?? Opts(), hierarchySource: null, ackIdentityDirectory: directory);
|
|
|
|
private sealed class ManualClock : TimeProvider
|
|
{
|
|
private DateTimeOffset _now = new(2026, 11, 9, 12, 0, 0, TimeSpan.Zero);
|
|
|
|
public void Advance(TimeSpan by) => _now += by;
|
|
|
|
public override DateTimeOffset GetUtcNow() => _now;
|
|
}
|
|
|
|
/// <summary>Metadata-only store: <see cref="ListAsync"/> is all the directory may call.</summary>
|
|
private sealed class FakeSecretStore : ISecretStore
|
|
{
|
|
private readonly Dictionary<string, bool> _rows = new(StringComparer.Ordinal); // name → deleted
|
|
private readonly Lock _lock = new();
|
|
private int _listCalls;
|
|
|
|
public bool Fail { get; set; }
|
|
|
|
public Action? OnList { get; set; }
|
|
|
|
public int ListCalls => Volatile.Read(ref _listCalls);
|
|
|
|
public void Add(string name)
|
|
{
|
|
lock (_lock) _rows[name] = false;
|
|
}
|
|
|
|
public void AddIdentity(string keyId)
|
|
{
|
|
Add(GalaxyAckIdentitySecrets.PasswordSecretName(keyId));
|
|
Add(GalaxyAckIdentitySecrets.UserSecretName(keyId));
|
|
}
|
|
|
|
public void RemoveIdentity(string keyId, bool softDelete)
|
|
{
|
|
lock (_lock)
|
|
{
|
|
foreach (var name in new[]
|
|
{
|
|
GalaxyAckIdentitySecrets.UserSecretName(keyId), GalaxyAckIdentitySecrets.PasswordSecretName(keyId),
|
|
})
|
|
{
|
|
if (softDelete) _rows[name] = true;
|
|
else _rows.Remove(name);
|
|
}
|
|
}
|
|
}
|
|
|
|
public Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct)
|
|
{
|
|
Interlocked.Increment(ref _listCalls);
|
|
OnList?.Invoke();
|
|
if (Fail) throw new InvalidOperationException("store unavailable");
|
|
lock (_lock)
|
|
{
|
|
// Deleted rows are returned regardless, to prove the directory filters them itself.
|
|
IReadOnlyList<SecretMetadata> rows = _rows.Select(r => new SecretMetadata
|
|
{
|
|
Name = new SecretName(r.Key),
|
|
ContentType = SecretContentType.Text,
|
|
KekId = "k",
|
|
Revision = 1,
|
|
IsDeleted = r.Value,
|
|
CreatedUtc = DateTimeOffset.UnixEpoch,
|
|
UpdatedUtc = DateTimeOffset.UnixEpoch,
|
|
}).ToList();
|
|
return Task.FromResult(rows);
|
|
}
|
|
}
|
|
|
|
public Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct) => throw Unexpected();
|
|
|
|
public Task UpsertAsync(StoredSecret row, CancellationToken ct) => throw Unexpected();
|
|
|
|
public Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct) => throw Unexpected();
|
|
|
|
public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) => throw Unexpected();
|
|
|
|
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct) => throw Unexpected();
|
|
|
|
public Task<bool> ApplyRewrapAsync(StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
|
|
throw Unexpected();
|
|
|
|
// The directory must read metadata only — never a row (that would decrypt-adjacent I/O per key).
|
|
private static InvalidOperationException Unexpected() => new("the directory may only list metadata");
|
|
}
|
|
}
|