fix(security): warn-only compatibility for deployed drivers + upgrade preflight (#815)

User decision: save, Test Connect, live browse and the deploy gate keep
refusing out-of-namespace credential references, but an ALREADY-DEPLOYED
driver must not fault at the upgrade restart or (Galaxy) at the next
gateway reconnect, since ReopenAsync re-resolves Gateway.ApiKeySecretRef.

- LegacyCredentialReferenceTracker (Core.Abstractions): the one, deprecated,
  call-site-chosen compatibility path. Running Galaxy (InitializeAsync,
  ReopenAsync, ack fallback) and OpcUaClient (session open) pass it and keep
  the pre-#815 semantics (case-insensitive prefixes, Galaxy file:, OpcUaClient
  literal pass-through) with a rate-limited Warning (driver/field/allowed
  form, never the value) and a credential-compat:<field> diagnostics entry
  (both drivers now implement IDriverDiagnostics). Browsers and probes pass
  none and stay hard-fail; Heidenhain has none.
- Deploy-gate message names the migration.
- Upgrade preflight: CredentialReferencePreflightService (admin nodes, once
  after ApplicationStarted, ConfigDb only, resolves nothing) logs a Warning
  per Draft row / Deployed-artifact finding; the /raw banner shows them plus
  Running drivers reporting warn-only use (driver-health diagnostics).
- docs/security.md: exact replacement table (incl. the old form default
  env:MX_API_KEY, file:, un-namespaced secret:, casing changes), the
  ReopenAsync reconnect behaviour, the warn-only window and its removal.

Co-Authored-By: Claude <noreply@anthropic.com>

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Contracts/GalaxySecretRef.cs

@@ -36,19 +36,32 @@ public static class GalaxySecretRef
     /// <param name="logger">Optional logger for warning on cleartext keys.</param>
     /// <param name="ct">Cancellation token for the async <c>secret:</c> resolution.</param>
     /// <param name="environment">Environment reader (tests); null ⇒ the process environment.</param>
+    /// <param name="legacy">
+    ///     ⚠️ Deprecated #815 compatibility: the RUNNING driver passes its tracker, so a pre-#815 out-of-namespace
+    ///     reference is resolved with the old semantics and warned about instead of failing. The AdminUI browser
+    ///     passes null (hard-fail). See <see cref="LegacyCredentialReferenceTracker"/>.
+    /// </param>
     /// <returns>The resolved API-key string.</returns>
     public static async Task<string> ResolveApiKeyAsync(
         string secretRef,
         ISecretResolver resolver,
         ILogger? logger = null,
         CancellationToken ct = default,
-        Func<string, string?>? environment = null)
+        Func<string, string?>? environment = null,
+        LegacyCredentialReferenceTracker? legacy = null)
     {
         ArgumentException.ThrowIfNullOrEmpty(secretRef);
         ArgumentNullException.ThrowIfNull(resolver);
 
         var scope = DriverCredentialPolicy.GalaxyScope;
-        if (CredentialReferences.Parse(secretRef, scope, out _) == CredentialReferenceKind.Literal)
+        var kind = CredentialReferences.Parse(secretRef, scope, out _);
+        if (kind == CredentialReferenceKind.OutOfScope && legacy is not null)
+        {
+            legacy.Report("Gateway.ApiKeySecretRef");
+            return await LegacyResolveApiKeyAsync(secretRef, resolver, logger, ct, environment).ConfigureAwait(false);
+        }
+
+        if (kind == CredentialReferenceKind.Literal)
         {
             if (secretRef.StartsWith(CredentialReferences.DevPrefix, StringComparison.OrdinalIgnoreCase))
             {
@@ -74,4 +87,49 @@ public static class GalaxySecretRef
             environment,
             ct).ConfigureAwait(false);
     }
+
+    /// <summary>
+    ///     ⚠️ DEPRECATED (lmxopcua#815 warn-only compatibility; delete in the hard-fail release): the pre-#815 resolver,
+    ///     verbatim — case-insensitive <c>env:</c> / <c>file:</c> / <c>dev:</c> / <c>secret:</c>, anything else a
+    ///     literal. Reached ONLY for an out-of-namespace reference in an already-deployed, running driver.
+    /// </summary>
+    private static async Task<string> LegacyResolveApiKeyAsync(
+        string secretRef, ISecretResolver resolver, ILogger? logger, CancellationToken ct, Func<string, string?>? environment)
+    {
+        if (secretRef.StartsWith("env:", StringComparison.OrdinalIgnoreCase))
+        {
+            var name = secretRef[4..];
+            var value = (environment ?? Environment.GetEnvironmentVariable)(name);
+            return !string.IsNullOrEmpty(value)
+                ? value
+                : throw new InvalidOperationException(
+                    $"Galaxy.Gateway.ApiKeySecretRef resolves to env var '{name}', but it is unset.");
+        }
+
+        if (secretRef.StartsWith("file:", StringComparison.OrdinalIgnoreCase))
+        {
+            var path = secretRef[5..];
+            if (!File.Exists(path))
+                throw new InvalidOperationException($"Galaxy.Gateway.ApiKeySecretRef points at '{path}', which doesn't exist.");
+            var contents = (await File.ReadAllTextAsync(path, ct).ConfigureAwait(false)).Trim();
+            return !string.IsNullOrEmpty(contents)
+                ? contents
+                : throw new InvalidOperationException($"Galaxy.Gateway.ApiKeySecretRef file '{path}' is empty.");
+        }
+
+        if (secretRef.StartsWith("dev:", StringComparison.OrdinalIgnoreCase)) return secretRef[4..];
+
+        if (secretRef.StartsWith("secret:", StringComparison.OrdinalIgnoreCase))
+        {
+            var name = secretRef["secret:".Length..];
+            var value = await resolver.GetAsync(new SecretName(name), ct).ConfigureAwait(false);
+            return !string.IsNullOrEmpty(value)
+                ? value
+                : throw new InvalidOperationException(
+                    $"Galaxy.Gateway.ApiKeySecretRef resolves secret '{name}', but it is absent from the store (fail-closed).");
+        }
+
+        logger?.LogWarning("Galaxy.Gateway.ApiKeySecretRef is being treated as a literal cleartext API key.");
+        return secretRef;
+    }
 }

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs

@@ -38,8 +38,17 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy;
 ///     </para>
 /// </remarks>
 public sealed class GalaxyDriver
-    : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IRediscoverable, IHostConnectivityProbe, IAlarmSource, IAlarmAcknowledgePreflight, ISkippedTagReporter, IDisposable, IAsyncDisposable
+    : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IRediscoverable, IHostConnectivityProbe, IAlarmSource, IAlarmAcknowledgePreflight, ISkippedTagReporter, IDriverDiagnostics, IDisposable, IAsyncDisposable
 {
+    // ⚠️ lmxopcua#815 warn-only compatibility (deprecated): this RUNNING driver keeps resolving a pre-#815
+    // out-of-namespace credential reference (API key on InitializeAsync AND every gateway ReopenAsync, plus the
+    // ack fallback password) with the old semantics, warning rate-limited and surfacing it on the diagnostics bag.
+    // The AdminUI browser and the probe never get a tracker, so they stay hard-fail.
+    private readonly LegacyCredentialReferenceTracker _legacyCredentials;
+
+    /// <inheritdoc />
+    IReadOnlyDictionary<string, string> IDriverDiagnostics.GetDiagnostics() => _legacyCredentials.Diagnostics;
+
     /// <summary>Authored tags this driver could not dial — a TagConfig with no usable
     /// <c>attributeRef</c> has no MXAccess address to bind.</summary>
     private readonly SkippedTagLedger _skippedTags = new();
@@ -342,6 +351,8 @@ public sealed class GalaxyDriver
         _options = options ?? throw new ArgumentNullException(nameof(options));
         _logger = logger ?? NullLogger<GalaxyDriver>.Instance;
         _secretResolver = secretResolver ?? NullSecretResolver.Instance;
+        _legacyCredentials = new LegacyCredentialReferenceTracker(
+            DriverCredentialPolicy.GalaxyScope, _driverInstanceId, message => _logger.LogWarning("{CredentialCompatWarning}", message));
         _hierarchySource = hierarchySource;
         _dataReader = dataReader;
         _dataWriter = dataWriter;
@@ -349,7 +360,7 @@ public sealed class GalaxyDriver
         _alarmAcknowledger = alarmAcknowledger;
         _alarmFeed = alarmFeed;
         _ackCredentialResolver = ackCredentialResolver
-            ?? new GalaxyAckCredentialResolver(_driverInstanceId, _options.AlarmAck, _secretResolver, _logger);
+            ?? new GalaxyAckCredentialResolver(_driverInstanceId, _options.AlarmAck, _secretResolver, _logger, legacy: _legacyCredentials);
 
         // Build the v3 RawPath ⇄ attributeRef tables from the authored raw tags. Empty when no tags
         // are authored (live-browse path), leaving every translation an identity no-op.
@@ -1098,7 +1109,9 @@ public sealed class GalaxyDriver
         }
     }
 
-    private async Task<MxGatewayClientOptions> BuildClientOptionsAsync(
+    // Internal: the ONE client-options build InitializeAsync, ReopenAsync (every gateway reconnect) and the
+    // repository/alarm clients share — the #815 warn-only tests drive it directly to prove a reconnect re-resolves.
+    internal async Task<MxGatewayClientOptions> BuildClientOptionsAsync(
         GalaxyGatewayOptions gw, CancellationToken cancellationToken)
     {
         // Resolve the API-key ref BEFORE the object initializer — the secret: arm is
@@ -1108,7 +1121,7 @@ public sealed class GalaxyDriver
         // (GalaxySecretRef) so the runtime driver and the AdminUI browser share one
         // implementation; the secret: arm resolves through the shared ISecretResolver.
         var apiKey = await GalaxySecretRef
-            .ResolveApiKeyAsync(gw.ApiKeySecretRef, _secretResolver, _logger, cancellationToken)
+            .ResolveApiKeyAsync(gw.ApiKeySecretRef, _secretResolver, _logger, cancellationToken, legacy: _legacyCredentials)
             .ConfigureAwait(false);
         return new MxGatewayClientOptions
         {

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GalaxyAckCredentialResolver.cs

@@ -22,6 +22,7 @@ internal sealed class GalaxyAckCredentialResolver : IGalaxyAckCredentialResolver
     private readonly ISecretResolver _secrets;
     private readonly ILogger _logger;
     private readonly Func<string, string?>? _environment;
+    private readonly LegacyCredentialReferenceTracker? _legacy;
 
     /// <summary>Initializes a new instance of the <see cref="GalaxyAckCredentialResolver"/> class.</summary>
     /// <param name="driverInstanceId">The owning driver instance id (for messages).</param>
@@ -29,11 +30,13 @@ internal sealed class GalaxyAckCredentialResolver : IGalaxyAckCredentialResolver
     /// <param name="secrets">The shared secret resolver.</param>
     /// <param name="logger">Logger; never receives a secret.</param>
     /// <param name="environment">Environment reader (tests); null ⇒ the process environment.</param>
+    /// <param name="legacy">⚠️ #815 warn-only compatibility tracker of the running driver (deprecated); null ⇒ hard-fail.</param>
     public GalaxyAckCredentialResolver(
         string driverInstanceId, GalaxyAlarmAckOptions? options, ISecretResolver secrets, ILogger logger,
-        Func<string, string?>? environment = null)
+        Func<string, string?>? environment = null, LegacyCredentialReferenceTracker? legacy = null)
     {
         _environment = environment;
+        _legacy = legacy;
         _driverInstanceId = driverInstanceId;
         _options = options ?? new GalaxyAlarmAckOptions();
         _secrets = secrets ?? throw new ArgumentNullException(nameof(secrets));
@@ -156,6 +159,13 @@ internal sealed class GalaxyAckCredentialResolver : IGalaxyAckCredentialResolver
     /// </summary>
     internal async Task<string?> ResolvePasswordRefAsync(string reference, CancellationToken ct)
     {
+        if (_legacy is not null
+            && CredentialReferences.Parse(reference, DriverCredentialPolicy.GalaxyScope, out _) == CredentialReferenceKind.OutOfScope)
+        {
+            _legacy.Report("AlarmAck.FallbackPasswordSecretRef");
+            return await LegacyResolvePasswordRefAsync(reference, ct).ConfigureAwait(false);
+        }
+
         var resolved = await CredentialReferences.ResolveOrNullAsync(
             reference,
             DriverCredentialPolicy.GalaxyScope,
@@ -169,6 +179,28 @@ internal sealed class GalaxyAckCredentialResolver : IGalaxyAckCredentialResolver
             : resolved;
     }
 
+    /// <summary>
+    ///     ⚠️ DEPRECATED (lmxopcua#815 warn-only compatibility; delete in the hard-fail release): the pre-#815 resolver,
+    ///     verbatim. Reached ONLY for an out-of-namespace reference in an already-deployed, running driver.
+    /// </summary>
+    private async Task<string?> LegacyResolvePasswordRefAsync(string reference, CancellationToken ct)
+    {
+        if (reference.StartsWith("${secret:", StringComparison.OrdinalIgnoreCase) && reference.EndsWith('}'))
+            reference = "secret:" + reference["${secret:".Length..^1];
+        if (reference.StartsWith("secret:", StringComparison.OrdinalIgnoreCase))
+            return await _secrets.GetAsync(new SecretName(reference["secret:".Length..]), ct).ConfigureAwait(false);
+        if (reference.StartsWith("env:", StringComparison.OrdinalIgnoreCase))
+            return (_environment ?? Environment.GetEnvironmentVariable)(reference[4..]);
+        if (reference.StartsWith("file:", StringComparison.OrdinalIgnoreCase))
+        {
+            var path = reference[5..];
+            return File.Exists(path) ? (await File.ReadAllTextAsync(path, ct).ConfigureAwait(false)).TrimEnd('\r', '\n') : null;
+        }
+
+        if (reference.StartsWith("dev:", StringComparison.OrdinalIgnoreCase)) return reference[4..];
+        throw new FormatException("AlarmAck.FallbackPasswordSecretRef must be a secret:/env:/dev: reference, not a literal.");
+    }
+
     // Safe-to-print form of a reference: the kind + name for secret/env/file, never the dev: value or a literal.
     internal static string DescribeRef(string reference) => CredentialReferences.Describe(reference);
 }

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Session/OpcUaUserIdentityFactory.cs

@@ -115,17 +115,22 @@ public static class OpcUaUserIdentityFactory
     /// <param name="resolver">The shared secret resolver, or null when none is available.</param>
     /// <param name="ct">Cancellation token.</param>
     /// <param name="environment">Environment reader (tests); null ⇒ the process environment.</param>
+    /// <param name="legacy">
+    ///     ⚠️ Deprecated #815 compatibility: the RUNNING driver's session open passes its tracker, so a pre-#815
+    ///     out-of-namespace value keeps the old semantics (a case-insensitive <c>secret:</c> resolves, anything else is
+    ///     a literal) with a rate-limited warning. The AdminUI browser passes null (hard-fail).
+    /// </param>
     /// <returns>An options copy whose credential fields carry resolved plaintext.</returns>
     public static async Task<OpcUaClientDriverOptions> ResolveSecretRefsAsync(
         OpcUaClientDriverOptions options, ISecretResolver? resolver, CancellationToken ct,
-        Func<string, string?>? environment = null)
+        Func<string, string?>? environment = null, LegacyCredentialReferenceTracker? legacy = null)
     {
         ArgumentNullException.ThrowIfNull(options);
 
-        var password = await ResolveFieldAsync(options.Password, nameof(options.Password), resolver, ct, environment)
+        var password = await ResolveFieldAsync(options.Password, nameof(options.Password), resolver, ct, environment, legacy)
             .ConfigureAwait(false);
         var certPassword = await ResolveFieldAsync(
-            options.UserCertificatePassword, nameof(options.UserCertificatePassword), resolver, ct, environment)
+            options.UserCertificatePassword, nameof(options.UserCertificatePassword), resolver, ct, environment, legacy)
             .ConfigureAwait(false);
 
         if (ReferenceEquals(password, options.Password)
@@ -139,12 +144,18 @@ public static class OpcUaUserIdentityFactory
 
     private static async Task<string?> ResolveFieldAsync(
         string? value, string fieldName, ISecretResolver? resolver, CancellationToken ct,
-        Func<string, string?>? environment)
+        Func<string, string?>? environment, LegacyCredentialReferenceTracker? legacy)
     {
         // A literal (or blank) passes through verbatim, reference-equal — the author already knows it.
         var kind = CredentialReferences.Parse(value, DriverCredentialPolicy.OpcUaClientScope, out _);
         if (kind is CredentialReferenceKind.Blank or CredentialReferenceKind.Literal) return value;
 
+        if (kind == CredentialReferenceKind.OutOfScope && legacy is not null)
+        {
+            legacy.Report(fieldName);
+            return await LegacyResolveFieldAsync(value!, fieldName, resolver, ct).ConfigureAwait(false);
+        }
+
         // lmxopcua#815: the value is sent to the operator-authored endpoint, so only secret:opcuaclient/… and
         // env:OTOPCUA_OPCUACLIENT_… resolve; anything else (out of scope, file:) throws BEFORE any store/env read.
         return await CredentialReferences.ResolveAsync(
@@ -156,4 +167,24 @@ public static class OpcUaUserIdentityFactory
             environment,
             ct).ConfigureAwait(false);
     }
+
+    /// <summary>
+    ///     ⚠️ DEPRECATED (lmxopcua#815 warn-only compatibility; delete in the hard-fail release): the pre-#815 resolver,
+    ///     verbatim — a case-insensitive <c>secret:</c> resolves (fail-closed), anything else passes through as a
+    ///     literal. Reached ONLY for an out-of-namespace value in an already-deployed, running driver.
+    /// </summary>
+    private static async Task<string?> LegacyResolveFieldAsync(
+        string value, string fieldName, ISecretResolver? resolver, CancellationToken ct)
+    {
+        if (!value.StartsWith(CredentialReferences.SecretPrefix, StringComparison.OrdinalIgnoreCase)) return value;
+        var name = value[CredentialReferences.SecretPrefix.Length..];
+        if (resolver is null)
+            throw new InvalidOperationException(
+                $"OpcUaClientDriverOptions.{fieldName} references secret '{name}', but no secret store is available here (fail-closed).");
+        var resolved = await resolver.GetAsync(new SecretName(name), ct).ConfigureAwait(false);
+        return !string.IsNullOrEmpty(resolved)
+            ? resolved
+            : throw new InvalidOperationException(
+                $"OpcUaClientDriverOptions.{fieldName} resolves secret '{name}', but it is absent from the store (fail-closed).");
+    }
 }

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/OpcUaClientDriver.cs

@@ -29,8 +29,16 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient;
 ///         back on its own recovers in place.
 ///     </para>
 /// </remarks>
-public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, ISkippedTagReporter, IDriverConnectionLossSource, IDisposable, IAsyncDisposable
+public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, ISkippedTagReporter, IDriverConnectionLossSource, IDriverDiagnostics, IDisposable, IAsyncDisposable
 {
+    // ⚠️ lmxopcua#815 warn-only compatibility (deprecated): the session open of this RUNNING driver keeps
+    // resolving a pre-#815 out-of-namespace credential with the old semantics, warning rate-limited and surfacing
+    // it on the diagnostics bag. The AdminUI browser never gets a tracker, so it stays hard-fail.
+    private readonly LegacyCredentialReferenceTracker _legacyCredentials;
+
+    /// <inheritdoc />
+    IReadOnlyDictionary<string, string> IDriverDiagnostics.GetDiagnostics() => _legacyCredentials.Diagnostics;
+
     /// <summary>Authored tags whose TagConfig carried no usable <c>nodeId</c>, so there is no upstream node
     /// to dial. This driver's skip path was the quietest of the twelve: <c>NamespaceMap</c>
     /// drops such an entry with a bare <c>continue</c> and not even a log line.</summary>
@@ -82,6 +90,8 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
         _diag = new OpcUaClientDiagnostics(driverInstanceId);
         _logger = logger ?? NullLogger<OpcUaClientDriver>.Instance;
         _secretResolver = secretResolver ?? NullSecretResolver.Instance;
+        _legacyCredentials = new LegacyCredentialReferenceTracker(
+            DriverCredentialPolicy.OpcUaClientScope, driverInstanceId, message => _logger.LogWarning("{CredentialCompatWarning}", message));
         _sdkTelemetry = logger is null ? SdkTelemetry.Context : SdkTelemetry.ForLogger(logger);
         PostRecoveryReconcileDelay = () => _options.Timeout + _options.KeepAliveInterval;
     }
@@ -554,7 +564,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
             // in BuildCertificateIdentity — flow through BuildUserIdentity(resolvedOptions), so a
             // single resolve covers both paths.
             var resolvedOptions = await OpcUaClientSecretResolution
-                .ResolveSecretRefsAsync(_options, _secretResolver, cancellationToken).ConfigureAwait(false);
+                .ResolveSecretRefsAsync(_options, _secretResolver, cancellationToken, _legacyCredentials).ConfigureAwait(false);
 
             var identity = BuildUserIdentity(resolvedOptions);
 

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/OpcUaClientSecretResolution.cs

@@ -31,16 +31,18 @@ internal static class OpcUaClientSecretResolution
     /// <param name="options">The raw driver options (credential fields may carry <c>secret:</c> refs).</param>
     /// <param name="resolver">The shared secret resolver used by the <c>secret:</c> arm.</param>
     /// <param name="ct">Cancellation token for the async secret resolution.</param>
+    /// <param name="legacy">⚠️ #815 warn-only compatibility tracker of the running driver (deprecated); null ⇒ hard-fail.</param>
     /// <returns>An options copy whose credential fields carry resolved plaintext.</returns>
     /// <exception cref="InvalidOperationException">
     ///     A <c>secret:</c> ref names a secret that is absent/tombstoned in the store (fail-closed).
     /// </exception>
     internal static Task<OpcUaClientDriverOptions> ResolveSecretRefsAsync(
-        OpcUaClientDriverOptions options, ISecretResolver resolver, CancellationToken ct)
+        OpcUaClientDriverOptions options, ISecretResolver resolver, CancellationToken ct,
+        Core.Abstractions.LegacyCredentialReferenceTracker? legacy = null)
     {
         ArgumentNullException.ThrowIfNull(options);
         ArgumentNullException.ThrowIfNull(resolver);
         // One authority, shared with the AdminUI live browse (Driver.OpcUaClient.Session).
-        return OpcUaUserIdentityFactory.ResolveSecretRefsAsync(options, resolver, ct);
+        return OpcUaUserIdentityFactory.ResolveSecretRefsAsync(options, resolver, ct, legacy: legacy);
     }
 }