fix(mesh): Batch B review fixes — bounded store read, cache poison guard, test hardening

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs

@@ -26,6 +26,14 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
 ///         the point at which central learns a cluster's membership (and so possibly its key) has
 ///         changed.
 ///     </para>
+///     <para>
+///         <b><c>Invalidate</c> is not a revocation barrier.</b> It drops the cached entry; it does
+///         not wait for, cancel, or supersede a resolve already in flight. A lookup that entered the
+///         store before the call can still publish and return the pre-rotation key to its caller, and
+///         a lookup racing the eviction can re-cache the old value it had already read. Key rotation
+///         is therefore eventually consistent by seconds, not atomic — treat <c>Invalidate</c> as
+///         "stop serving this from memory", never as "this key is now dead everywhere".
+///     </para>
 /// </remarks>
 public interface IMeshPskProvider
 {
@@ -54,6 +62,14 @@ public sealed class MeshPskProvider : IMeshPskProvider
     /// </summary>
     public const string SecretNamePrefix = "OT-MESH-PSK-";
 
+    /// <summary>
+    ///     Ceiling on one store read. Not a tuning knob — a backstop: the read runs under the
+    ///     <see cref="Lazy{T}"/> publication lock, so a store that never answers would park the
+    ///     calling gRPC thread <i>and</i> every other caller queued behind it for that cluster.
+    ///     Expiry fails closed like any other fault.
+    /// </summary>
+    private static readonly TimeSpan StoreReadTimeout = TimeSpan.FromSeconds(3);
+
     private readonly IOptions<MeshTransportOptions> _options;
     private readonly ISecretResolver _secretResolver;
     private readonly ILogger<MeshPskProvider> _logger;
@@ -118,7 +134,27 @@ public sealed class MeshPskProvider : IMeshPskProvider
                 () => self.ResolveFromStore(id), LazyThreadSafetyMode.ExecutionAndPublication),
             this);
 
-        var resolved = entry.Value;
+        string? resolved;
+        try
+        {
+            resolved = entry.Value;
+        }
+        catch
+        {
+            // ResolveFromStore swallows everything the STORE can throw — but it logs while doing so,
+            // and a logger provider that throws escapes the factory. Lazy under
+            // ExecutionAndPublication caches that exception and rethrows it on every later read, so
+            // without this guard one bad log call would deny the cluster for the life of the process
+            // (nothing calls Invalidate until Phase 1A). Evict instance-keyed, same as the negative
+            // path below, and fail closed.
+            //
+            // Deliberately silent: the most likely reason we are here is that logging is what threw,
+            // so logging the failure is the one action with a real chance of throwing again — out of
+            // an authentication gate, turning a denial into a 500.
+            _cache.TryRemove(new KeyValuePair<string, Lazy<string?>>(clusterId, entry));
+            return false;
+        }
+
         if (string.IsNullOrWhiteSpace(resolved))
         {
             // Never retain a negative. A store that was briefly unreachable would otherwise stay
@@ -154,9 +190,12 @@ public sealed class MeshPskProvider : IMeshPskProvider
     ///     <para>
     ///         Blocking on the async resolver is deliberate. The consumer is a gRPC server
     ///         interceptor, whose <c>Authorize</c> is synchronous in all four sibling interceptors,
-    ///         and making this seam async would push that shape onto every one of them. The block
-    ///         happens at most once per cluster per process — every later lookup is a dictionary hit
-    ///         — and the store is a node-local SQLite file, not a network call.
+    ///         and making this seam async would push that shape onto every one of them. What makes
+    ///         the block acceptable is that it is <b>bounded</b> (<see cref="StoreReadTimeout"/>) and
+    ///         happens at most once per cluster per process — every later lookup is a dictionary hit.
+    ///         The bound does not assume anything about what backs the store: this code cannot see
+    ///         whether the resolver reads a local file or crosses a network, so it refuses to wait
+    ///         forever on either.
     ///     </para>
     ///     <para>
     ///         The catch is broad on purpose: this is an authentication boundary, and the set of
@@ -184,12 +223,15 @@ public sealed class MeshPskProvider : IMeshPskProvider
             return null;
         }
 
+        using var cts = new CancellationTokenSource(StoreReadTimeout);
         try
         {
-            return _secretResolver.GetAsync(name, CancellationToken.None).GetAwaiter().GetResult();
+            return _secretResolver.GetAsync(name, cts.Token).GetAwaiter().GetResult();
         }
         catch (Exception ex)
         {
+            // Covers the timeout too — a cancelled read arrives here as OperationCanceledException
+            // and is treated as every other fault is: no key.
             _logger.LogWarning(
                 ex,
                 "Failed to resolve the mesh PSK for cluster {ClusterId} from secret {SecretName}. "