fix(inbound-api): revision-check compiled handler cache against the fresh ApiMethod row — heals failover/direct-SQL/known-bad staleness

Also updates the now-obsolete CompileAndRegister_NonCompilingUpdate test
(InboundScriptExecutorTests.cs, not in the plan's Files list) to assert the
deliberate DB-authoritative behavior: a broken save now 500s consistently on
every node instead of the stale delegate silently serving.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj

src/ZB.MOM.WW.ScadaBridge.InboundAPI/InboundScriptExecutor.cs

@@ -20,23 +20,35 @@ public class InboundScriptExecutor
 {
     private readonly ILogger<InboundScriptExecutor> _logger;
 
+    // A cached compiled handler, paired with the exact script text it was
+    // compiled from. The DB row is authoritative: on every request the cached
+    // Script is ordinal-compared against the freshly-fetched ApiMethod.Script; a
+    // mismatch means the cache is stale (a standby node compiled its own copy at
+    // startup, or the row was edited via direct SQL / bundle import) and the
+    // handler is recompiled in place. A null Script means "no revision info,
+    // serve as-is" — the RegisterHandler test seam, exempt from revision checks.
+    private sealed record CachedHandler(string? Script, Func<InboundScriptContext, Task<object?>> Handler);
+
     // This executor is registered as a singleton and its handler cache
     // is read and written from concurrent ASP.NET request threads. A plain Dictionary is
     // not safe for concurrent read/write, so a ConcurrentDictionary is used throughout.
-    private readonly ConcurrentDictionary<string, Func<InboundScriptContext, Task<object?>>> _scriptHandlers = new();
+    private readonly ConcurrentDictionary<string, CachedHandler> _scriptHandlers = new();
 
     // A script that fails to compile (or violates the trust model)
-    // is recorded here so it is compiled at most once. Without this, every subsequent
+    // is recorded here — keyed by method name, valued by the exact script text
+    // that failed — so it is compiled at most once. Without this, every subsequent
     // request for a broken method re-runs the expensive Roslyn compilation — a CPU
-    // amplification vector since the inbound API has no rate limiting. The entry is
-    // cleared whenever the method is (re)compiled via CompileAndRegister.
+    // amplification vector since the inbound API has no rate limiting. A record only
+    // short-circuits when the CURRENT script text is the one that failed: a changed
+    // script always gets a fresh compile. The entry is cleared whenever the method
+    // is (re)compiled successfully.
     //
     // Bound the cache so a spam attack of unique method names cannot
     // grow it without bound. Once the cap is reached new bad-method records are
     // dropped — the cache is just a fast-fail optimisation; the per-request DB
     // lookup remains the correctness path.
     private const int KnownBadMethodsCap = 1000;
-    private readonly ConcurrentDictionary<string, byte> _knownBadMethods = new();
+    private readonly ConcurrentDictionary<string, string> _knownBadMethods = new();
 
     /// <summary>
     /// Diagnostic helper — returns the current size of the
@@ -46,20 +58,27 @@ public class InboundScriptExecutor
     internal int KnownBadMethodCount => _knownBadMethods.Count;
 
     /// <summary>
-    /// Records <paramref name="methodName"/> in the known-bad-methods
-    /// cache only if the cache has not reached <see cref="KnownBadMethodsCap"/>.
-    /// Once full, new records are dropped (paying the cheap recompile next time
-    /// rather than leaking memory under a unique-name flood). Existing entries are
-    /// not touched — they remain capped fast-fail records until cleared on a
-    /// successful (re)compile in <see cref="CompileAndRegister"/>.
+    /// Records <paramref name="methodName"/> → <paramref name="script"/> in the
+    /// known-bad-methods cache so the CURRENT failing script text short-circuits the
+    /// next request. An existing record for the method is overwritten so a newer bad
+    /// script replaces an older one (the fast-fail always tracks the latest failure).
+    /// A brand-new record is only added while the cache has not reached
+    /// <see cref="KnownBadMethodsCap"/>; once full, new method names are dropped
+    /// (paying the cheap recompile next time rather than leaking memory under a
+    /// unique-name flood). Records are cleared on a successful (re)compile.
     /// </summary>
-    private void TryRecordBadMethod(string methodName)
+    private void TryRecordBadMethod(string methodName, string script)
     {
+        // Overwrite an existing record (newer bad script wins) without counting
+        // against the cap; only a genuinely new method name is gated by the cap.
         if (_knownBadMethods.ContainsKey(methodName))
+        {
+            _knownBadMethods[methodName] = script;
             return;
+        }
         if (_knownBadMethods.Count >= KnownBadMethodsCap)
             return;
-        _knownBadMethods.TryAdd(methodName, 0);
+        _knownBadMethods.TryAdd(methodName, script);
     }
 
     private readonly IServiceProvider _serviceProvider;
@@ -76,13 +95,18 @@ public class InboundScriptExecutor
     }
 
     /// <summary>
-    /// Registers a compiled script handler for a method name.
+    /// Registers a compiled script handler for a method name. This is the script-less
+    /// test seam: the handler is stored with no associated script text, so it is
+    /// <b>exempt from the per-request revision check</b> and serves as-is regardless of
+    /// the fetched row's script. Production compiles always flow through
+    /// <see cref="CompileAndRegister(ApiMethod)"/>, which records the script text and is
+    /// revision-checked.
     /// </summary>
     /// <param name="methodName">The method name to register.</param>
     /// <param name="handler">The compiled handler function.</param>
     public void RegisterHandler(string methodName, Func<InboundScriptContext, Task<object?>> handler)
     {
-        _scriptHandlers[methodName] = handler;
+        _scriptHandlers[methodName] = new CachedHandler(null, handler);
     }
 
     /// <summary>
@@ -106,11 +130,14 @@ public class InboundScriptExecutor
     /// Compiles and registers a single API method script, additionally reporting the
     /// compilation diagnostics. Returns <c>false</c> together with the human-readable
     /// error list if the script is empty, fails Roslyn compilation, or violates the
-    /// script trust model. On failure the <b>previously registered handler (if any) is
-    /// left in place</b> — a broken save never replaces a working delegate — so the
-    /// management Create/Update path can surface a non-fatal "saved but did not compile"
-    /// warning while the prior version keeps serving requests (see
-    /// <c>HandleCreateApiMethod</c>/<c>HandleUpdateApiMethod</c>).
+    /// script trust model. On failure this method does not overwrite any previously
+    /// registered handler, but the persisted DB row is authoritative: <see cref="ExecuteAsync"/>
+    /// revision-checks the cached handler against the freshly-fetched
+    /// <c>ApiMethod.Script</c> on every request, so once a broken script is saved the
+    /// method returns a compilation error consistently on every node rather than the
+    /// stale delegate silently serving. The management Create/Update path can therefore
+    /// surface a "saved but did not compile" warning that reflects real, consistent
+    /// behavior (see <c>HandleCreateApiMethod</c>/<c>HandleUpdateApiMethod</c>).
     /// </summary>
     /// <param name="method">The API method to compile and register.</param>
     /// <param name="errors">On failure, the compilation/trust-model error messages; empty on success.</param>
@@ -125,7 +152,7 @@ public class InboundScriptExecutor
             // through the capped TryRecordBadMethod helper so the cache
             // cannot grow without bound under a flood of unique method names.
             errors = compileErrors;
-            TryRecordBadMethod(method.Name);
+            TryRecordBadMethod(method.Name, method.Script ?? string.Empty);
             return false;
         }
 
@@ -133,12 +160,12 @@ public class InboundScriptExecutor
         // failure record so a fixed script is no longer treated as bad.
         errors = Array.Empty<string>();
         _knownBadMethods.TryRemove(method.Name, out _);
-        return Register(method.Name, handler);
+        return Register(method, handler);
     }
 
-    private bool Register(string methodName, Func<InboundScriptContext, Task<object?>> handler)
+    private bool Register(ApiMethod method, Func<InboundScriptContext, Task<object?>> handler)
     {
-        _scriptHandlers[methodName] = handler;
+        _scriptHandlers[method.Name] = new CachedHandler(method.Script, handler);
         return true;
     }
 
@@ -308,29 +335,43 @@ public class InboundScriptExecutor
                 dbHelper,
                 cts.Token);
 
-            if (!_scriptHandlers.TryGetValue(method.Name, out var handler))
+            _scriptHandlers.TryGetValue(method.Name, out var cached);
+            // Revision check: a cached handler compiled from different script text is stale
+            // (standby-after-failover, direct SQL edit, bundle import). Null Script = test seam, exempt.
+            if (cached is { Script: not null }
+                && !string.Equals(cached.Script, method.Script, StringComparison.Ordinal))
             {
-                // A method already known to fail compilation must not
-                // be recompiled on every request — short-circuit before Roslyn runs.
-                if (_knownBadMethods.ContainsKey(method.Name))
+                _logger.LogInformation(
+                    "API method {Method} script changed since it was cached; recompiling", method.Name);
+                cached = null;
+            }
+
+            if (cached == null)
+            {
+                // Fast-fail only when the CURRENT script text is the one already known bad.
+                if (_knownBadMethods.TryGetValue(method.Name, out var badScript)
+                    && string.Equals(badScript, method.Script, StringComparison.Ordinal))
                     return new InboundScriptResult(false, null, "Script compilation failed for this method");
 
-                // Lazy compile on first request (handles methods created after startup).
-                // Compile outside the cache so a failed compile is not stored, then add
-                // atomically so concurrent first-callers share a single handler instance.
+                // Lazy compile (handles methods created after startup, and stale/known-bad
+                // recompiles). Compile outside the cache so a failed compile is not stored,
+                // then add so concurrent first-callers share a single handler instance.
                 var (compiled, _) = Compile(method);
                 if (compiled == null)
                 {
                     // Cache the failure so the next request short-circuits above.
                     // Routed through TryRecordBadMethod so the
                     // cache is bounded under a flood of unique method names.
-                    TryRecordBadMethod(method.Name);
+                    TryRecordBadMethod(method.Name, method.Script ?? string.Empty);
                     return new InboundScriptResult(false, null, "Script compilation failed for this method");
                 }
-                handler = _scriptHandlers.GetOrAdd(method.Name, compiled);
+
+                _knownBadMethods.TryRemove(method.Name, out _);
+                cached = new CachedHandler(method.Script, compiled);
+                _scriptHandlers[method.Name] = cached; // last-write-wins; concurrent first-callers race benignly
             }
 
-            var result = await handler(context).WaitAsync(cts.Token);
+            var result = await cached.Handler(context).WaitAsync(cts.Token);
 
             var resultJson = result != null
                 ? JsonSerializer.Serialize(result)