fix(alarms): refuse a keyed native ack with no Galaxy identity synchronously, from a cached key set (lmxopcua#765)

Item 3 of #765. A keyed (API-key) session whose key had no Galaxy ack identity
got Good and then a revert, because whether the key has an identity lives in
the Secrets store, which must not be read under the SDK lock.

GalaxyAckIdentityDirectory (Driver.Galaxy) keeps a node-wide set of the key ids
that have a galaxy-ack-user secret. It lists this node's Secrets store
(metadata only, nothing decrypted) and is read lock-free.
GalaxyDriver.GetAcknowledgeRefusal (IAlarmAcknowledgePreflight, published
through NativeAlarmAckPreflightView as in #760) now refuses a keyed session
whose key the directory reports Absent, when the driver has no fallback ack
account. A refusal also asks for a refresh.

Fails open: before the first refresh, once the snapshot is older than
StaleAfter, and from a signalled identity-secret change until the next
refresh, the lookup answers Unknown and the ack takes the old Good-then-revert
path.

Refresh: GalaxyAckIdentityRefreshService runs on driver nodes, every
Drivers:Galaxy:AckIdentityCache:RefreshIntervalSeconds (default 30;
StaleAfterSeconds 0 = three intervals; validated at start). It also refreshes
early on secret changes. The Secrets library has no change event, so
SecretsRegistration decorates ISecretCacheInvalidator, which is called by
AdminUI /api-keys, /admin/secrets and the replication appliers. API-key
changes need no event: a revoked key is already denied before the pre-check.

Tests: Galaxy.Tests GalaxyAckIdentityDirectoryTests (16) and Host.Tests
GalaxyAckIdentityCacheWiringTests (9). Seven separate mutations each turn at
least one test red: pre-check ignores the directory, factory drops it, no
invalidator decoration, no staleness check, no change-pending check, no
refresh on refusal, loop never wakes early.
Docs: Galaxy.md, AlarmTracking.md, Configuration.md, .claude/rules/alarms.md.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs

@@ -119,6 +119,9 @@ public sealed class GalaxyDriver
     // lmxopcua#749: resolves the Galaxy account each acknowledge authenticates as (API key identity →
     // fallback account → refused). Built in the ctor — it needs only options + the secret resolver.
     private readonly IGalaxyAckCredentialResolver _ackCredentialResolver;
+    // lmxopcua#765: the cached key → has-Galaxy-identity set the synchronous ack pre-check reads (no I/O).
+    // Null = no directory wired, so a keyed session is never pre-refused (the pre-#765 behaviour).
+    private readonly IGalaxyAckIdentityDirectory? _ackIdentityDirectory;
     private IGalaxyAlarmFeed? _alarmFeed;
     private readonly Lock _alarmHandlersLock = new();
     private readonly Lock _alarmFeedLock = new();
@@ -262,15 +265,21 @@ public sealed class GalaxyDriver
     /// <param name="options">The Galaxy driver configuration options.</param>
     /// <param name="secretResolver">Resolves the <c>secret:</c> arm of <c>Gateway.ApiKeySecretRef</c>.</param>
     /// <param name="logger">Optional logger instance for diagnostics.</param>
+    /// <param name="ackIdentityDirectory">
+    ///     Optional cached set of API keys with a Galaxy ack identity (lmxopcua#765), consulted by the synchronous
+    ///     ack pre-check. Null = keyed sessions are never pre-refused.
+    /// </param>
     public GalaxyDriver(
         string driverInstanceId,
         GalaxyDriverOptions options,
         ISecretResolver secretResolver,
-        ILogger<GalaxyDriver>? logger = null)
+        ILogger<GalaxyDriver>? logger = null,
+        IGalaxyAckIdentityDirectory? ackIdentityDirectory = null)
         : this(driverInstanceId, options,
               hierarchySource: null, dataReader: null, dataWriter: null, subscriber: null,
               alarmAcknowledger: null, alarmFeed: null, logger,
-              secretResolver: secretResolver ?? throw new ArgumentNullException(nameof(secretResolver)))
+              secretResolver: secretResolver ?? throw new ArgumentNullException(nameof(secretResolver)),
+              ackIdentityDirectory: ackIdentityDirectory)
     {
     }
 
@@ -307,6 +316,7 @@ public sealed class GalaxyDriver
     ///     Optional ack-credential seam (lmxopcua#749). Defaults to <see cref="GalaxyAckCredentialResolver"/>
     ///     over <see cref="GalaxyDriverOptions.AlarmAck"/> and <paramref name="secretResolver"/>.
     /// </param>
+    /// <param name="ackIdentityDirectory">Optional cached key → Galaxy-identity set (lmxopcua#765).</param>
     internal GalaxyDriver(
         string driverInstanceId,
         GalaxyDriverOptions options,
@@ -320,8 +330,10 @@ public sealed class GalaxyDriver
         ISecretResolver? secretResolver = null,
         IGalaxyDeployWatchSource? deployWatchSource = null,
         IGalaxyPlatformSource? platformSource = null,
-        IGalaxyAckCredentialResolver? ackCredentialResolver = null)
+        IGalaxyAckCredentialResolver? ackCredentialResolver = null,
+        IGalaxyAckIdentityDirectory? ackIdentityDirectory = null)
     {
+        _ackIdentityDirectory = ackIdentityDirectory;
         _deployWatchSource = deployWatchSource;
         _platformSource = platformSource;
         _driverInstanceId = !string.IsNullOrWhiteSpace(driverInstanceId)
@@ -1968,17 +1980,32 @@ public sealed class GalaxyDriver
 
     /// <inheritdoc />
     /// <remarks>
-    ///     Only the case decidable without I/O (lmxopcua#760): an anonymous session (no API key, so no key-mapped
-    ///     Galaxy identity can exist) on a driver with no <c>AlarmAck.FallbackUser</c>. A keyed session is let
-    ///     through — whether the key has an identity lives in the Secrets store, which must not be read under the
-    ///     SDK lock; <see cref="AcknowledgeAsync"/> still refuses it, and the host reverts the condition.
+    ///     Only cases decidable without I/O, on a driver with no <c>AlarmAck.FallbackUser</c>: an anonymous session
+    ///     (lmxopcua#760 — no API key, so no key-mapped Galaxy identity can exist), and a keyed session whose key the
+    ///     <see cref="IGalaxyAckIdentityDirectory"/> reports <see cref="GalaxyAckIdentityPresence.Absent"/>
+    ///     (lmxopcua#765 — a cached, periodically refreshed view of the Secrets store, because the store itself must
+    ///     not be read under the SDK lock). Anything the directory does not know for certain (no directory, no
+    ///     refresh yet, stale, a change pending) is let through: <see cref="AcknowledgeAsync"/> still refuses it,
+    ///     and the host reverts the condition. A refusal also asks the directory to refresh, so a retry after an
+    ///     identity was set out of band (the <c>secret</c> CLI) is not refused for a whole refresh interval.
     /// </remarks>
     public string? GetAcknowledgeRefusal(string? apiKeyId)
     {
-        if (!string.IsNullOrWhiteSpace(apiKeyId) || (_options.AlarmAck?.HasFallback ?? false)) return null;
-        return $"the session is anonymous (no API key) and Galaxy driver '{_driverInstanceId}' has no " +
-            "AlarmAck.FallbackUser configured, so the acknowledge cannot be sent to Galaxy. Connect with an API key " +
-            "that has a Galaxy ack identity (AdminUI /api-keys) or configure the driver's fallback ack account.";
+        if (_options.AlarmAck?.HasFallback ?? false) return null;
+        if (string.IsNullOrWhiteSpace(apiKeyId))
+        {
+            return $"the session is anonymous (no API key) and Galaxy driver '{_driverInstanceId}' has no " +
+                "AlarmAck.FallbackUser configured, so the acknowledge cannot be sent to Galaxy. Connect with an API key " +
+                "that has a Galaxy ack identity (AdminUI /api-keys) or configure the driver's fallback ack account.";
+        }
+
+        if (_ackIdentityDirectory is not { } directory
+            || directory.GetPresence(apiKeyId) != GalaxyAckIdentityPresence.Absent) return null;
+
+        directory.RequestRefresh();
+        return $"API key '{apiKeyId.Trim()}' has no Galaxy ack identity and Galaxy driver '{_driverInstanceId}' has no " +
+            "AlarmAck.FallbackUser configured, so the acknowledge cannot be sent to Galaxy. Map a Galaxy ack identity " +
+            "to the key (AdminUI /api-keys) or configure the driver's fallback ack account.";
     }
 
     private static string PrincipalOf(AlarmAcknowledgeRequest ack) =>

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriverFactoryExtensions.cs

@@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 using ZB.MOM.WW.OtOpcUa.Core.Hosting;
 using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
 using ZB.MOM.WW.Secrets.Abstractions;
 
 namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy;
@@ -31,14 +32,20 @@ public static class GalaxyDriverFactoryExtensions
     ///     each instance's <c>Gateway.ApiKeySecretRef</c>.
     /// </param>
     /// <param name="loggerFactory">The optional logger factory for creating drivers.</param>
+    /// <param name="ackIdentityDirectory">
+    ///     Optional node-wide cached set of API keys with a Galaxy ack identity (lmxopcua#765), shared by every
+    ///     Galaxy driver instance for the synchronous ack pre-check. Null = keyed sessions are never pre-refused.
+    /// </param>
     public static void Register(
         DriverFactoryRegistry registry,
         ISecretResolver secretResolver,
-        ILoggerFactory? loggerFactory = null)
+        ILoggerFactory? loggerFactory = null,
+        IGalaxyAckIdentityDirectory? ackIdentityDirectory = null)
     {
         ArgumentNullException.ThrowIfNull(registry);
         ArgumentNullException.ThrowIfNull(secretResolver);
-        registry.Register(DriverTypeName, (id, json) => CreateInstance(id, json, loggerFactory, secretResolver));
+        registry.Register(DriverTypeName,
+            (id, json) => CreateInstance(id, json, loggerFactory, secretResolver, ackIdentityDirectory));
     }
 
     /// <summary>
@@ -58,9 +65,11 @@ public static class GalaxyDriverFactoryExtensions
     /// <param name="driverConfigJson">The driver configuration in JSON format.</param>
     /// <param name="loggerFactory">The optional logger factory for creating drivers.</param>
     /// <param name="secretResolver">The secret resolver for the driver's <c>secret:</c> API-key arm.</param>
+    /// <param name="ackIdentityDirectory">Optional cached key → Galaxy-identity set (lmxopcua#765).</param>
     /// <returns>The constructed Galaxy driver instance.</returns>
     public static GalaxyDriver CreateInstance(
-        string driverInstanceId, string driverConfigJson, ILoggerFactory? loggerFactory, ISecretResolver secretResolver)
+        string driverInstanceId, string driverConfigJson, ILoggerFactory? loggerFactory, ISecretResolver secretResolver,
+        IGalaxyAckIdentityDirectory? ackIdentityDirectory = null)
     {
         ArgumentNullException.ThrowIfNull(secretResolver);
         ArgumentException.ThrowIfNullOrWhiteSpace(driverInstanceId);
@@ -108,7 +117,8 @@ public static class GalaxyDriverFactoryExtensions
                 FallbackPasswordSecretRef: NullIfBlank(dto.AlarmAck?.FallbackPasswordSecretRef)),
         };
 
-        return new GalaxyDriver(driverInstanceId, options, secretResolver, loggerFactory?.CreateLogger<GalaxyDriver>());
+        return new GalaxyDriver(
+            driverInstanceId, options, secretResolver, loggerFactory?.CreateLogger<GalaxyDriver>(), ackIdentityDirectory);
     }
 
     private static string? NullIfBlank(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim();

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyAckIdentityDirectory.cs

@@ -0,0 +1,216 @@
+using System.Collections.Frozen;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
+using ZB.MOM.WW.Secrets.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
+
+/// <summary>What the <see cref="IGalaxyAckIdentityDirectory"/> knows about one API key's Galaxy ack identity.</summary>
+public enum GalaxyAckIdentityPresence
+{
+    /// <summary>
+    ///     Not known right now — no successful refresh yet, the last one is stale, a change is pending, or the key
+    ///     id is not one the directory can look up. Callers must treat it as "may have one" (fail open).
+    /// </summary>
+    Unknown = 0,
+
+    /// <summary>The key has a Galaxy ack user secret (it may still be refused at ack time, e.g. no password).</summary>
+    Present,
+
+    /// <summary>The key has no Galaxy ack user secret as of a fresh refresh.</summary>
+    Absent,
+}
+
+/// <summary>
+///     Lock-free answer to "does this API key have a Galaxy ack identity?" (lmxopcua#765), for the synchronous
+///     native-ack pre-check (<see cref="ZB.MOM.WW.OtOpcUa.Core.Abstractions.IAlarmAcknowledgePreflight"/>), which
+///     runs under the OPC UA SDK lock and must not read the Secrets store.
+/// </summary>
+public interface IGalaxyAckIdentityDirectory
+{
+    /// <summary>Looks the key up in the cached snapshot. No I/O, never throws.</summary>
+    /// <param name="apiKeyId">The session's API key id.</param>
+    /// <returns>The key's presence, or <see cref="GalaxyAckIdentityPresence.Unknown"/> when not known.</returns>
+    GalaxyAckIdentityPresence GetPresence(string? apiKeyId);
+
+    /// <summary>Asks for a refresh soon (coalesced; never blocks). Used after a refusal so a retry sees a new identity.</summary>
+    void RequestRefresh();
+}
+
+/// <summary>
+///     Production <see cref="IGalaxyAckIdentityDirectory"/>: a periodically refreshed set of the API key ids that
+///     have a Galaxy ack user secret (<see cref="GalaxyAckIdentitySecrets.UserSecretName"/>) in this node's
+///     Secrets store — the same store <see cref="GalaxyAckCredentialResolver"/> reads at ack time.
+/// </summary>
+/// <remarks>
+///     <para>
+///         Built from <see cref="ISecretStore.ListAsync"/> metadata only — nothing is decrypted, no password is
+///         read. A refresh runs every <c>refreshInterval</c> (<see cref="RunAsync"/>), and early on
+///         <see cref="NotifySecretChanged"/> / <see cref="RequestRefresh"/>.
+///     </para>
+///     <para>
+///         ⚠️ Fails OPEN: anything not certain answers <see cref="GalaxyAckIdentityPresence.Unknown"/>, which
+///         lets the ack through to today's path (the driver resolves at ack time and the host reverts a refusal).
+///         That covers no refresh yet, a store with nothing to list, a snapshot older than <c>staleAfter</c>
+///         (e.g. the store has been failing), and a change to an identity secret signalled since the snapshot
+///         was taken — so an identity just set through AdminUI or arriving by replication is never refused in
+///         the window before the next refresh. A change the process is not told about (the <c>secret</c> CLI
+///         writing the store directly) is picked up by the next timed refresh, or the refresh a refusal kicks.
+///     </para>
+/// </remarks>
+public sealed class GalaxyAckIdentityDirectory : IGalaxyAckIdentityDirectory, IDisposable
+{
+    private const string UserSuffix = "/galaxy-ack-user";
+
+    private readonly ISecretStore? _store;
+    private readonly TimeProvider _time;
+    private readonly ILogger _logger;
+    private readonly SemaphoreSlim _wake = new(0, 1);
+
+    // Bumped on every signalled change to an identity secret; a snapshot taken under an older value is not trusted.
+    private long _changeGeneration;
+    private volatile Snapshot? _snapshot;
+
+    /// <summary>Initializes a new instance of the <see cref="GalaxyAckIdentityDirectory"/> class.</summary>
+    /// <param name="store">This node's Secrets store; null = nothing to list, so every lookup is Unknown.</param>
+    /// <param name="refreshInterval">How often <see cref="RunAsync"/> refreshes.</param>
+    /// <param name="staleAfter">How old a snapshot may get before lookups answer Unknown.</param>
+    /// <param name="timeProvider">Clock for snapshot age; defaults to the system clock.</param>
+    /// <param name="logger">Logger; never receives a secret value.</param>
+    public GalaxyAckIdentityDirectory(
+        ISecretStore? store, TimeSpan refreshInterval, TimeSpan staleAfter,
+        TimeProvider? timeProvider = null, ILogger? logger = null)
+    {
+        if (refreshInterval <= TimeSpan.Zero)
+            throw new ArgumentOutOfRangeException(nameof(refreshInterval), refreshInterval, "Must be positive.");
+        if (staleAfter <= TimeSpan.Zero)
+            throw new ArgumentOutOfRangeException(nameof(staleAfter), staleAfter, "Must be positive.");
+        _store = store;
+        RefreshInterval = refreshInterval;
+        StaleAfter = staleAfter;
+        _time = timeProvider ?? TimeProvider.System;
+        _logger = logger ?? NullLogger.Instance;
+    }
+
+    /// <summary>Gets the refresh period of <see cref="RunAsync"/>.</summary>
+    public TimeSpan RefreshInterval { get; }
+
+    /// <summary>Gets the snapshot age after which every lookup answers Unknown.</summary>
+    public TimeSpan StaleAfter { get; }
+
+    /// <inheritdoc />
+    public GalaxyAckIdentityPresence GetPresence(string? apiKeyId)
+    {
+        var snapshot = _snapshot;
+        if (snapshot is null) return GalaxyAckIdentityPresence.Unknown;
+        if (snapshot.Generation != Interlocked.Read(ref _changeGeneration)) return GalaxyAckIdentityPresence.Unknown;
+        if (_time.GetUtcNow() - snapshot.RefreshedUtc > StaleAfter) return GalaxyAckIdentityPresence.Unknown;
+        if (TryNormalize(apiKeyId) is not { } keyId) return GalaxyAckIdentityPresence.Unknown;
+        return snapshot.KeyIds.Contains(keyId) ? GalaxyAckIdentityPresence.Present : GalaxyAckIdentityPresence.Absent;
+    }
+
+    /// <inheritdoc />
+    public void RequestRefresh()
+    {
+        try
+        {
+            _wake.Release();
+        }
+        catch (SemaphoreFullException)
+        {
+            // A refresh is already requested — coalesced.
+        }
+        catch (ObjectDisposedException)
+        {
+            // Shutting down.
+        }
+    }
+
+    /// <summary>
+    ///     Tells the directory a secret changed (wired from the Secrets cache-invalidation seam, which AdminUI
+    ///     writes, the Secrets UI and replication all call). A Galaxy ack identity secret marks the current
+    ///     snapshot untrusted until the next refresh, and requests one; any other name is ignored.
+    /// </summary>
+    /// <param name="name">The changed secret.</param>
+    public void NotifySecretChanged(SecretName name)
+    {
+        if (!name.Value.StartsWith(GalaxyAckIdentitySecrets.Prefix, StringComparison.Ordinal)) return;
+        Interlocked.Increment(ref _changeGeneration);
+        RequestRefresh();
+    }
+
+    /// <summary>Lists the store once and replaces the snapshot. Never throws except on cancellation.</summary>
+    /// <param name="cancellationToken">Cancels the listing.</param>
+    /// <returns>True when the snapshot was replaced; false on a failure (the old snapshot keeps ageing).</returns>
+    public async Task<bool> RefreshAsync(CancellationToken cancellationToken)
+    {
+        if (_store is null) return false;
+        // Read BEFORE listing: a change signalled while the listing runs leaves this snapshot untrusted.
+        var generation = Interlocked.Read(ref _changeGeneration);
+        try
+        {
+            var rows = await _store.ListAsync(includeDeleted: false, cancellationToken).ConfigureAwait(false);
+            var keyIds = new HashSet<string>(StringComparer.Ordinal);
+            foreach (var row in rows)
+            {
+                if (row.IsDeleted) continue;
+                var name = row.Name.Value;
+                if (!name.StartsWith(GalaxyAckIdentitySecrets.Prefix, StringComparison.Ordinal)
+                    || !name.EndsWith(UserSuffix, StringComparison.Ordinal)
+                    || name.Length <= GalaxyAckIdentitySecrets.Prefix.Length + UserSuffix.Length) continue;
+                if (TryNormalize(name[GalaxyAckIdentitySecrets.Prefix.Length..^UserSuffix.Length]) is { } keyId)
+                    keyIds.Add(keyId);
+            }
+
+            _snapshot = new Snapshot(keyIds.ToFrozenSet(StringComparer.Ordinal), _time.GetUtcNow(), generation);
+            return true;
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            _logger.LogWarning(
+                "Galaxy ack identity directory: listing the Secrets store failed ({ExceptionType}); native acks from " +
+                "keyed sessions fall back to the ack-time check once the snapshot is older than {StaleAfter}",
+                ex.GetType().Name, StaleAfter);
+            return false;
+        }
+    }
+
+    /// <summary>Refreshes now, then every <see cref="RefreshInterval"/> or sooner when asked, until cancelled.</summary>
+    /// <param name="cancellationToken">Stops the loop.</param>
+    /// <returns>A task that completes when cancelled.</returns>
+    public async Task RunAsync(CancellationToken cancellationToken)
+    {
+        try
+        {
+            while (!cancellationToken.IsCancellationRequested)
+            {
+                await RefreshAsync(cancellationToken).ConfigureAwait(false);
+                await _wake.WaitAsync(RefreshInterval, cancellationToken).ConfigureAwait(false);
+            }
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            // Normal shutdown.
+        }
+    }
+
+    /// <inheritdoc />
+    public void Dispose() => _wake.Dispose();
+
+    // Same shape GalaxyAckIdentitySecrets accepts (trimmed, lower-case ASCII letters/digits); anything else is
+    // not a key id the resolver could look up, so the directory does not claim to know about it.
+    private static string? TryNormalize(string? apiKeyId)
+    {
+        if (string.IsNullOrWhiteSpace(apiKeyId)) return null;
+        var id = apiKeyId.Trim().ToLowerInvariant();
+        foreach (var c in id)
+        {
+            if (!char.IsAsciiLetterOrDigit(c)) return null;
+        }
+
+        return id;
+    }
+
+    private sealed record Snapshot(FrozenSet<string> KeyIds, DateTimeOffset RefreshedUtc, long Generation);
+}