feat(central): one central cache-invalidation topic; role-mapping and external-definition caches follow peer edits (ScadaBridge#100 item 3, ScadaBridge#83)
The #96 site-psk-invalidation topic was the first cross-node invalidation broadcast. It is generalised into ONE central DistributedPubSub topic, central-cache-invalidation, carrying typed messages (Commons ICentralCacheInvalidation: SitePskInvalidated, RoleMappingsInvalidated, ExternalDefinitionsInvalidated, InboundApiMethodInvalidated) in an envelope stamped with the publisher's process-unique origin id. - Host: CentralCacheInvalidationBus (ICentralCacheInvalidationPublisher, no-op until the mediator is set, never throws) and CentralCacheInvalidationSubscriber, which skips its own node's envelopes and hands the peer's to every registered ICentralCacheInvalidationHandler (a failing handler does not stop the rest). Replaces SitePskInvalidationSubscriber. - Migration of #96: SitePskRotationInvalidator publishes through the bus and is the PSK handler; SitePskInvalidated keeps its type name and is also published bare on site-psk-invalidation, which the subscriber still listens to, so a mixed-version central pair keeps exchanging PSK invalidations. Remove after one release. - Audit D1: RoleMappingCache is a handler; ManagementActor's post-commit role-mapping invalidation also publishes RoleMappingsInvalidated, so a revocation is immediate on the peer instead of after RoleMappingCacheSeconds. - Audit D3 / #83 (peer half): ExternalDefinitionLookupCache is a handler; the 9 ManagementActor handlers and the Transport bundle importer publish ExternalDefinitionsInvalidated after commit. (#83's local import half was already in.) TTLs remain the fallback for a lost broadcast or a direct SQL edit. Tests: bus and subscriber unit tests (origin filter, legacy topic), peer-node tests through the real caches in ManagementService.Tests, importer publish, composition root, and a real two-node cluster from the production HOCON (CentralCacheInvalidationTwoNodeTests) proving every message type crosses with the production serializer. Docs: Host, Communication, Security, ManagementService, ESG, and the #85 cache audit. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
+33
@@ -0,0 +1,33 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Communication;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Tells the peer central node to drop a cache entry this node has just invalidated
|
||||
/// (ScadaBridge#100 item 3). Central only; resolved optionally, so a host without it (a site
|
||||
/// node, a plain test host) simply keeps invalidations local.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Callers invalidate their own node's cache first and then publish: the publishing node does
|
||||
/// not apply its own broadcast. Publishing never throws; before the actor system is up it is a
|
||||
/// no-op, and the peer's cache TTL bounds the staleness exactly as it did before.
|
||||
/// </remarks>
|
||||
public interface ICentralCacheInvalidationPublisher
|
||||
{
|
||||
/// <summary>Broadcasts <paramref name="invalidation"/> to the other central node.</summary>
|
||||
/// <param name="invalidation">The typed invalidation.</param>
|
||||
void Publish(ICentralCacheInvalidation invalidation);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies a peer's <see cref="ICentralCacheInvalidation"/> to one of this node's caches. Each
|
||||
/// cache-owning component registers one (as an enumerable service) and ignores the message types
|
||||
/// it does not own; the central invalidation subscriber hands every received message to all of
|
||||
/// them.
|
||||
/// </summary>
|
||||
public interface ICentralCacheInvalidationHandler
|
||||
{
|
||||
/// <summary>Applies <paramref name="invalidation"/> if this handler owns its type.</summary>
|
||||
/// <param name="invalidation">The received invalidation.</param>
|
||||
void Apply(ICentralCacheInvalidation invalidation);
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// Marker for a typed cache-invalidation message exchanged between the two central nodes over
|
||||
/// the single central <c>DistributedPubSub</c> topic <c>central-cache-invalidation</c>
|
||||
/// (ScadaBridge#100 item 3, #83, #103). Each central node caches configuration-database state
|
||||
/// in-process; a change made through one node drops that node's cache directly and publishes
|
||||
/// one of these so the peer drops its copy too, instead of waiting out the cache TTL.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Delivery is best-effort and at-most-once: every cache that takes part still has a TTL or a
|
||||
/// per-request revision check as its correctness fallback. Message contracts follow the
|
||||
/// additive-only evolution rule — a node that does not know a message type ignores it.
|
||||
/// </remarks>
|
||||
public interface ICentralCacheInvalidation;
|
||||
|
||||
/// <summary>
|
||||
/// The LDAP group → role mapping table or a Deployer site-scope rule changed: drop the
|
||||
/// <c>RoleMappingCache</c> snapshot (audit D1).
|
||||
/// </summary>
|
||||
public sealed record RoleMappingsInvalidated : ICentralCacheInvalidation;
|
||||
|
||||
/// <summary>
|
||||
/// An external system, external-system method or database connection definition changed
|
||||
/// (management edit or Transport bundle import): drop every <c>ExternalDefinitionLookupCache</c>
|
||||
/// entry (audit D3, #83).
|
||||
/// </summary>
|
||||
public sealed record ExternalDefinitionsInvalidated : ICentralCacheInvalidation;
|
||||
|
||||
/// <summary>
|
||||
/// An inbound API method was created, updated, deleted or recompiled: drop the method's
|
||||
/// compiled handler, its known-bad compile record and its cached row (#103).
|
||||
/// </summary>
|
||||
/// <param name="MethodName">The API method's name.</param>
|
||||
public sealed record InboundApiMethodInvalidated(string MethodName) : ICentralCacheInvalidation;
|
||||
|
||||
/// <summary>
|
||||
/// What travels on the <c>central-cache-invalidation</c> topic: the invalidation plus the
|
||||
/// publishing node's process-unique origin id, so a node can skip its own broadcast (it already
|
||||
/// invalidated locally before publishing).
|
||||
/// </summary>
|
||||
/// <param name="OriginId">Process-unique id of the publishing node's invalidation bus.</param>
|
||||
/// <param name="Invalidation">The typed invalidation.</param>
|
||||
public sealed record CentralCacheInvalidationEnvelope(string OriginId, ICentralCacheInvalidation Invalidation);
|
||||
@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.ExternalSystems;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Communication;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ExternalSystemGateway;
|
||||
|
||||
@@ -20,7 +21,7 @@ namespace ZB.MOM.WW.ScadaBridge.ExternalSystemGateway;
|
||||
/// deleted one is visible within one TTL.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ExternalDefinitionLookupCache : IExternalDefinitionCacheInvalidator
|
||||
public sealed class ExternalDefinitionLookupCache : IExternalDefinitionCacheInvalidator, ICentralCacheInvalidationHandler
|
||||
{
|
||||
private sealed record Entry(object? Value, DateTimeOffset ExpiresAtUtc);
|
||||
|
||||
@@ -75,10 +76,20 @@ public sealed class ExternalDefinitionLookupCache : IExternalDefinitionCacheInva
|
||||
/// <summary>Drops every cached definition.</summary>
|
||||
/// <remarks>
|
||||
/// Called post-commit by the ManagementActor edit handlers (Gitea #65) and by the Transport
|
||||
/// bundle importer through <see cref="IExternalDefinitionCacheInvalidator"/> (Gitea #83).
|
||||
/// bundle importer through <see cref="IExternalDefinitionCacheInvalidator"/> (Gitea #83). Both
|
||||
/// also publish <see cref="ExternalDefinitionsInvalidated"/> so the peer central node drops its
|
||||
/// copy through <see cref="Apply"/> (ScadaBridge#100 item 3).
|
||||
/// </remarks>
|
||||
public void InvalidateAll() => _entries.Clear();
|
||||
|
||||
/// <summary>Applies a peer central node's <see cref="ExternalDefinitionsInvalidated"/>.</summary>
|
||||
/// <param name="invalidation">The received invalidation; other types are ignored.</param>
|
||||
public void Apply(ICentralCacheInvalidation invalidation)
|
||||
{
|
||||
if (invalidation is ExternalDefinitionsInvalidated)
|
||||
InvalidateAll();
|
||||
}
|
||||
|
||||
private async Task<T?> GetAsync<T>(string key, Func<Task<T?>> fetch) where T : class
|
||||
{
|
||||
if (!Enabled)
|
||||
|
||||
@@ -48,6 +48,10 @@ public static class ServiceCollectionExtensions
|
||||
// importer can drop it after a committed import without referencing this assembly.
|
||||
services.TryAddSingleton<IExternalDefinitionCacheInvalidator>(
|
||||
sp => sp.GetRequiredService<ExternalDefinitionLookupCache>());
|
||||
// Audit D3 / #83 (ScadaBridge#100 item 3): an edit or import made through the other central
|
||||
// node reaches this node's cache over the central cache-invalidation topic.
|
||||
services.TryAddEnumerable(ServiceDescriptor.Singleton<ICentralCacheInvalidationHandler, ExternalDefinitionLookupCache>(
|
||||
sp => sp.GetRequiredService<ExternalDefinitionLookupCache>()));
|
||||
services.AddScoped<ExternalSystemClient>();
|
||||
services.AddScoped<IExternalSystemClient>(sp => sp.GetRequiredService<ExternalSystemClient>());
|
||||
services.AddScoped<DatabaseGateway>();
|
||||
|
||||
@@ -463,16 +463,23 @@ akka {{
|
||||
"CentralControlGrpcService readiness set (service bound: {Bound})",
|
||||
centralControlGrpc is not null);
|
||||
|
||||
// Peer half of site-PSK rotation invalidation (Gitea #96): subscribe to the topic, and give
|
||||
// the invalidator the mediator so a rotation written on this node reaches the other one.
|
||||
var pskRotation = _serviceProvider.GetService<SitePskRotationInvalidator>();
|
||||
if (pskRotation is not null)
|
||||
// Peer half of central cache invalidation (ScadaBridge#100 item 3, generalising the #96
|
||||
// site-PSK topic): subscribe this node's cache handlers (site PSK, role mappings, external
|
||||
// definitions, inbound API methods) to the shared topic, and give the bus the mediator so
|
||||
// an invalidation made on this node reaches the other one.
|
||||
var cacheInvalidationBus = _serviceProvider.GetService<CentralCacheInvalidationBus>();
|
||||
if (cacheInvalidationBus is not null)
|
||||
{
|
||||
var handlers = _serviceProvider
|
||||
.GetServices<ICentralCacheInvalidationHandler>()
|
||||
.ToList();
|
||||
_actorSystem!.ActorOf(
|
||||
Props.Create(() => new SitePskInvalidationSubscriber(pskRotation)),
|
||||
"site-psk-invalidation");
|
||||
pskRotation.SetMediator(
|
||||
Props.Create(() => new CentralCacheInvalidationSubscriber(cacheInvalidationBus, handlers)),
|
||||
"central-cache-invalidation");
|
||||
cacheInvalidationBus.SetMediator(
|
||||
Akka.Cluster.Tools.PublishSubscribe.DistributedPubSub.Get(_actorSystem!).Mediator);
|
||||
_logger.LogInformation(
|
||||
"Central cache invalidation subscribed ({HandlerCount} handlers)", handlers.Count);
|
||||
}
|
||||
|
||||
// Wire up the CommunicationService with the actor reference
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Communication;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
|
||||
/// <summary>
|
||||
/// Central's cross-node cache-invalidation channel (ScadaBridge#100 item 3): one
|
||||
/// <c>DistributedPubSub</c> topic, <see cref="Topic"/>, carrying typed
|
||||
/// <see cref="ICentralCacheInvalidation"/> messages inside a
|
||||
/// <see cref="CentralCacheInvalidationEnvelope"/>. It generalises the per-cache topic #96 added
|
||||
/// for site PSK rotation; the role-mapping cache (audit D1), the external-definition cache
|
||||
/// (audit D3 / #83), the inbound API compile state (#103) and the site PSK cache all ride it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Origin filtering.</b> A publisher has already invalidated its own caches, and a node
|
||||
/// receives its own broadcast too. Re-applying is harmless for most caches but not free for all
|
||||
/// of them (the inbound API would drop a handler it has just compiled, and recompile it on the
|
||||
/// next request), so each envelope carries this bus's process-unique <see cref="OriginId"/> and
|
||||
/// the subscriber skips its own.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Migration from <c>site-psk-invalidation</c>.</b> A <see cref="SitePskInvalidated"/> is also
|
||||
/// published, bare, on <see cref="LegacySitePskTopic"/>, and the subscriber still listens there,
|
||||
/// so a central pair that is briefly mixed-version during an upgrade keeps exchanging PSK
|
||||
/// invalidations. Applying a PSK invalidation twice is harmless. Remove both once every central
|
||||
/// node runs this build.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class CentralCacheInvalidationBus : ICentralCacheInvalidationPublisher
|
||||
{
|
||||
/// <summary>The single central cache-invalidation topic.</summary>
|
||||
public const string Topic = "central-cache-invalidation";
|
||||
|
||||
/// <summary>The #96 per-cache topic, kept for mixed-version central pairs (see remarks).</summary>
|
||||
public const string LegacySitePskTopic = "site-psk-invalidation";
|
||||
|
||||
private readonly ILogger<CentralCacheInvalidationBus> _logger;
|
||||
private volatile IActorRef? _mediator;
|
||||
|
||||
/// <summary>Creates the bus.</summary>
|
||||
/// <param name="logger">Logger.</param>
|
||||
public CentralCacheInvalidationBus(ILogger<CentralCacheInvalidationBus> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>This process's origin id, stamped on every envelope it publishes.</summary>
|
||||
public string OriginId { get; } = Guid.NewGuid().ToString("N");
|
||||
|
||||
/// <summary>
|
||||
/// Supplies the <c>DistributedPubSub</c> mediator once the actor system exists. Until then
|
||||
/// publishing is a no-op (callers have already invalidated locally).
|
||||
/// </summary>
|
||||
/// <param name="mediator">The mediator to publish to.</param>
|
||||
public void SetMediator(IActorRef mediator) => _mediator = mediator;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Publish(ICentralCacheInvalidation invalidation)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(invalidation);
|
||||
var mediator = _mediator;
|
||||
if (mediator is null)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"No pub-sub mediator yet; {Invalidation} stays local to this node.", invalidation.GetType().Name);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
mediator.Tell(new Publish(Topic, new CentralCacheInvalidationEnvelope(OriginId, invalidation)));
|
||||
if (invalidation is SitePskInvalidated psk)
|
||||
{
|
||||
mediator.Tell(new Publish(LegacySitePskTopic, psk));
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Never let a broadcast failure fail the write that triggered it: the peer's TTL
|
||||
// (or its per-request revision check) still bounds the staleness.
|
||||
_logger.LogWarning(ex, "Publishing {Invalidation} to the peer central node failed.", invalidation.GetType().Name);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="envelope"/> was published by this process (and was therefore
|
||||
/// already applied locally).
|
||||
/// </summary>
|
||||
/// <param name="envelope">The received envelope.</param>
|
||||
/// <returns><c>true</c> for this node's own broadcast.</returns>
|
||||
public bool IsOwn(CentralCacheInvalidationEnvelope envelope) =>
|
||||
string.Equals(envelope.OriginId, OriginId, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Central actor that subscribes to <see cref="CentralCacheInvalidationBus.Topic"/> (and the
|
||||
/// legacy <see cref="CentralCacheInvalidationBus.LegacySitePskTopic"/>) and hands every
|
||||
/// invalidation published by the peer central node to this node's
|
||||
/// <see cref="ICentralCacheInvalidationHandler"/>s.
|
||||
/// </summary>
|
||||
public sealed class CentralCacheInvalidationSubscriber : ReceiveActor
|
||||
{
|
||||
private readonly IReadOnlyList<ICentralCacheInvalidationHandler> _handlers;
|
||||
|
||||
/// <summary>Creates the subscriber.</summary>
|
||||
/// <param name="bus">This node's bus, used to recognise this node's own broadcasts.</param>
|
||||
/// <param name="handlers">This node's cache handlers.</param>
|
||||
public CentralCacheInvalidationSubscriber(
|
||||
CentralCacheInvalidationBus bus, IReadOnlyList<ICentralCacheInvalidationHandler> handlers)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(bus);
|
||||
ArgumentNullException.ThrowIfNull(handlers);
|
||||
_handlers = handlers;
|
||||
|
||||
Receive<CentralCacheInvalidationEnvelope>(envelope =>
|
||||
{
|
||||
if (!bus.IsOwn(envelope) && envelope.Invalidation is not null)
|
||||
{
|
||||
Apply(envelope.Invalidation);
|
||||
}
|
||||
});
|
||||
// Bare message on the #96 topic, from a peer that predates the shared topic.
|
||||
Receive<SitePskInvalidated>(Apply);
|
||||
Receive<SubscribeAck>(_ => { });
|
||||
}
|
||||
|
||||
private void Apply(ICentralCacheInvalidation invalidation)
|
||||
{
|
||||
foreach (var handler in _handlers)
|
||||
{
|
||||
try
|
||||
{
|
||||
handler.Apply(invalidation);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// One failing cache must not stop the others; its own TTL still bounds it.
|
||||
Context.GetLogger().Warning(ex, "Applying peer invalidation {0} in {1} failed.",
|
||||
invalidation.GetType().Name, handler.GetType().Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
try
|
||||
{
|
||||
var mediator = DistributedPubSub.Get(Context.System).Mediator;
|
||||
mediator.Tell(new Subscribe(CentralCacheInvalidationBus.Topic, Self));
|
||||
mediator.Tell(new Subscribe(CentralCacheInvalidationBus.LegacySitePskTopic, Self));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Non-clustered host (TestKit): peer invalidation is simply unavailable.
|
||||
Context.GetLogger().Debug("DistributedPubSub not available — peer cache invalidation disabled: {0}", ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>DI wiring for <see cref="CentralCacheInvalidationBus"/>.</summary>
|
||||
public static class CentralCacheInvalidationServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Registers the central cache-invalidation bus as <see cref="ICentralCacheInvalidationPublisher"/>.
|
||||
/// Central only; the actor system wires the mediator and the subscriber at startup.
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection.</param>
|
||||
/// <returns>The same collection.</returns>
|
||||
public static IServiceCollection AddCentralCacheInvalidation(this IServiceCollection services)
|
||||
{
|
||||
services.TryAddSingleton<CentralCacheInvalidationBus>();
|
||||
services.TryAddSingleton<ICentralCacheInvalidationPublisher>(
|
||||
sp => sp.GetRequiredService<CentralCacheInvalidationBus>());
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -413,7 +413,10 @@ try
|
||||
builder.Services.AddScadaBridgeSecrets(builder.Configuration, SecretsNodeRole.Central);
|
||||
// An in-place rotation of a site's SB-GRPC-PSK-{siteId} through /admin/secrets must drop
|
||||
// central's cached copy of that key on BOTH central nodes (Gitea #96). Wraps the library's
|
||||
// ISecretCacheInvalidator; the peer half is wired in AkkaHostedService.RegisterCentralActors.
|
||||
// ISecretCacheInvalidator; the peer half rides the central cache-invalidation topic
|
||||
// (AddCentralCacheInvalidation, ScadaBridge#100 item 3 — also used by the role-mapping,
|
||||
// external-definition and inbound-API caches), wired in AkkaHostedService.RegisterCentralActors.
|
||||
builder.Services.AddCentralCacheInvalidation();
|
||||
builder.Services.AddSitePskRotationInvalidation();
|
||||
// Secrets UI authorization: adds the named policies secrets:manage + secrets:reveal
|
||||
// (role-based) consumed by the /admin/secrets page. AddSecretsAuthorization only ADDS
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.Secrets;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
@@ -26,9 +25,9 @@ namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
/// <para>
|
||||
/// <b>Why the peer needs telling.</b> Both central nodes cache the key in their own
|
||||
/// <see cref="SitePskProvider"/> (and their own resolver cache), and the write only ran on one of
|
||||
/// them. The peer is told over the central <c>DistributedPubSub</c> mediator on
|
||||
/// <see cref="Topic"/> — the same mediator the peer health replication uses. A node receives its
|
||||
/// own broadcast too; applying it twice is harmless, so there is no origin filtering.
|
||||
/// them. The peer is told with a <see cref="SitePskInvalidated"/> on the central cache-invalidation
|
||||
/// topic (<see cref="CentralCacheInvalidationBus"/>, ScadaBridge#100 item 3, which generalised
|
||||
/// the #96 <c>site-psk-invalidation</c> topic); this class is also the peer's handler for it.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Writes made outside this process (the <c>secret</c> CLI, a direct SQL write) never reach this
|
||||
@@ -36,23 +35,22 @@ namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
/// <c>ControlPlaneCredentials.WithSiteCredentials</c>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class SitePskRotationInvalidator : ISecretCacheInvalidator
|
||||
public sealed class SitePskRotationInvalidator : ISecretCacheInvalidator, ICentralCacheInvalidationHandler
|
||||
{
|
||||
/// <summary>The <c>DistributedPubSub</c> topic the central nodes exchange PSK invalidations on.</summary>
|
||||
public const string Topic = "site-psk-invalidation";
|
||||
|
||||
private readonly ISecretCacheInvalidator _inner;
|
||||
private readonly ISitePskProvider _pskProvider;
|
||||
private readonly ICentralCacheInvalidationPublisher? _publisher;
|
||||
private readonly ILogger<SitePskRotationInvalidator> _logger;
|
||||
private volatile IActorRef? _mediator;
|
||||
|
||||
/// <summary>Creates the invalidator.</summary>
|
||||
/// <param name="inner">The library's own invalidator (the secret resolver's cache).</param>
|
||||
/// <param name="pskProvider">Central's per-site PSK cache.</param>
|
||||
/// <param name="publisher">The central cache-invalidation publisher; null keeps evictions local.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
public SitePskRotationInvalidator(
|
||||
ISecretCacheInvalidator inner,
|
||||
ISitePskProvider pskProvider,
|
||||
ICentralCacheInvalidationPublisher? publisher,
|
||||
ILogger<SitePskRotationInvalidator> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(inner);
|
||||
@@ -60,16 +58,10 @@ public sealed class SitePskRotationInvalidator : ISecretCacheInvalidator
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
_inner = inner;
|
||||
_pskProvider = pskProvider;
|
||||
_publisher = publisher;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Supplies the <c>DistributedPubSub</c> mediator once the actor system exists. Until then
|
||||
/// (and on a host without one) evictions stay local.
|
||||
/// </summary>
|
||||
/// <param name="mediator">The mediator to publish peer invalidations to.</param>
|
||||
public void SetMediator(IActorRef mediator) => _mediator = mediator;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Invalidate(SecretName name)
|
||||
{
|
||||
@@ -85,12 +77,21 @@ public sealed class SitePskRotationInvalidator : ISecretCacheInvalidator
|
||||
_logger.LogInformation(
|
||||
"gRPC preshared key secret '{SecretName}' was written; dropped the cached key for site "
|
||||
+ "{SiteId} and notified the peer central node.", name.Value, siteId);
|
||||
_mediator?.Tell(new Publish(Topic, new SitePskInvalidated(siteId)));
|
||||
_publisher?.Publish(new SitePskInvalidated(siteId));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Apply(ICentralCacheInvalidation invalidation)
|
||||
{
|
||||
if (invalidation is SitePskInvalidated psk)
|
||||
{
|
||||
ApplyPeerInvalidation(psk.SiteId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies an invalidation received from a central node (the peer, or this node's own
|
||||
/// broadcast): evicts the resolver's cached secret and the PSK cache entry. Never re-publishes.
|
||||
/// Applies an invalidation received from the peer central node: evicts the resolver's cached
|
||||
/// secret and the PSK cache entry. Never re-publishes.
|
||||
/// </summary>
|
||||
/// <param name="siteId">The site whose key was rotated.</param>
|
||||
public void ApplyPeerInvalidation(string siteId)
|
||||
@@ -107,51 +108,22 @@ public sealed class SitePskRotationInvalidator : ISecretCacheInvalidator
|
||||
|
||||
/// <summary>
|
||||
/// Peer message: the gRPC preshared key of <paramref name="SiteId"/> was rotated on a central node.
|
||||
/// Rides the central cache-invalidation topic; kept in this namespace because the #96
|
||||
/// <c>site-psk-invalidation</c> topic carries it bare by this type name (see
|
||||
/// <see cref="CentralCacheInvalidationBus"/>).
|
||||
/// </summary>
|
||||
/// <param name="SiteId">The site whose key changed.</param>
|
||||
public sealed record SitePskInvalidated(string SiteId);
|
||||
|
||||
/// <summary>
|
||||
/// Central actor that subscribes to <see cref="SitePskRotationInvalidator.Topic"/> and applies
|
||||
/// each <see cref="SitePskInvalidated"/> to this node's caches.
|
||||
/// </summary>
|
||||
public sealed class SitePskInvalidationSubscriber : ReceiveActor
|
||||
{
|
||||
private readonly SitePskRotationInvalidator _invalidator;
|
||||
|
||||
/// <summary>Creates the subscriber.</summary>
|
||||
/// <param name="invalidator">This node's invalidator.</param>
|
||||
public SitePskInvalidationSubscriber(SitePskRotationInvalidator invalidator)
|
||||
{
|
||||
_invalidator = invalidator;
|
||||
Receive<SitePskInvalidated>(m => _invalidator.ApplyPeerInvalidation(m.SiteId));
|
||||
Receive<SubscribeAck>(_ => { });
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
try
|
||||
{
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(
|
||||
new Subscribe(SitePskRotationInvalidator.Topic, Self));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Non-clustered host (TestKit): peer invalidation is simply unavailable.
|
||||
Context.GetLogger().Debug("DistributedPubSub not available — peer PSK invalidation disabled: {0}", ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
public sealed record SitePskInvalidated(string SiteId) : ICentralCacheInvalidation;
|
||||
|
||||
/// <summary>DI wiring for <see cref="SitePskRotationInvalidator"/>.</summary>
|
||||
public static class SitePskRotationServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Replaces the secrets library's <see cref="ISecretCacheInvalidator"/> with
|
||||
/// <see cref="SitePskRotationInvalidator"/> wrapping the library's resolver. Central only;
|
||||
/// call after <c>AddScadaBridgeSecrets</c> and after <see cref="ISitePskProvider"/> is
|
||||
/// registered.
|
||||
/// <see cref="SitePskRotationInvalidator"/> wrapping the library's resolver, and registers it
|
||||
/// as a central cache-invalidation handler. Central only; call after
|
||||
/// <c>AddScadaBridgeSecrets</c>, after <see cref="ISitePskProvider"/> is registered and after
|
||||
/// <see cref="CentralCacheInvalidationServiceCollectionExtensions.AddCentralCacheInvalidation"/>.
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection.</param>
|
||||
/// <returns>The same collection.</returns>
|
||||
@@ -160,9 +132,12 @@ public static class SitePskRotationServiceCollectionExtensions
|
||||
services.TryAddSingleton(sp => new SitePskRotationInvalidator(
|
||||
sp.GetRequiredService<DefaultSecretResolver>(),
|
||||
sp.GetRequiredService<ISitePskProvider>(),
|
||||
sp.GetService<ICentralCacheInvalidationPublisher>(),
|
||||
sp.GetRequiredService<ILogger<SitePskRotationInvalidator>>()));
|
||||
services.Replace(ServiceDescriptor.Singleton<ISecretCacheInvalidator>(
|
||||
sp => sp.GetRequiredService<SitePskRotationInvalidator>()));
|
||||
services.AddSingleton<ICentralCacheInvalidationHandler>(
|
||||
sp => sp.GetRequiredService<SitePskRotationInvalidator>());
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Security;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
|
||||
@@ -2386,15 +2387,22 @@ public class ManagementActor : ReceiveActor
|
||||
/// call on this node resolves the fresh definition rather than waiting out
|
||||
/// <c>DefinitionCacheTtl</c> (ScadaBridge#65). Queued post-commit: invalidating before the commit
|
||||
/// would let a concurrent lookup re-cache the pre-edit row. The cache is per-node, so the other
|
||||
/// central node still converges within <c>DefinitionCacheTtl</c>. A no-op when the cache is not
|
||||
/// registered (site roles, plain test hosts).
|
||||
/// central node is told with an <see cref="ExternalDefinitionsInvalidated"/> on the central
|
||||
/// cache-invalidation topic (audit D3 / #83, ScadaBridge#100 item 3). A no-op when neither the
|
||||
/// cache nor the publisher is registered (site roles, plain test hosts).
|
||||
/// </summary>
|
||||
private static Task InvalidateExternalDefinitionCacheAfterCommit(IServiceProvider sp)
|
||||
{
|
||||
var cache = sp.GetService<ExternalDefinitionLookupCache>();
|
||||
return cache is null
|
||||
var peers = sp.GetService<ICentralCacheInvalidationPublisher>();
|
||||
return cache is null && peers is null
|
||||
? Task.CompletedTask
|
||||
: AfterConfigurationCommit(sp, () => { cache.InvalidateAll(); return Task.CompletedTask; });
|
||||
: AfterConfigurationCommit(sp, () =>
|
||||
{
|
||||
cache?.InvalidateAll();
|
||||
peers?.Publish(new ExternalDefinitionsInvalidated());
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
private static async Task<object?> HandleListExternalSystems(IServiceProvider sp)
|
||||
@@ -2902,15 +2910,23 @@ public class ManagementActor : ReceiveActor
|
||||
/// Drops this node's <see cref="RoleMappingCache"/> snapshot once a mapping / scope-rule
|
||||
/// change has committed, so the next role mapping on this node sees it immediately rather
|
||||
/// than after the cache TTL (lane K review M1). Queued post-commit: invalidating before the
|
||||
/// commit would let a concurrent mapping re-cache the pre-change table. Other central
|
||||
/// nodes still converge within <c>RoleMappingCacheSeconds</c>.
|
||||
/// commit would let a concurrent mapping re-cache the pre-change table. The peer central node
|
||||
/// is told with a <see cref="RoleMappingsInvalidated"/> on the central cache-invalidation topic
|
||||
/// (audit D1, ScadaBridge#100 item 3), so a revocation takes effect there at once too;
|
||||
/// <c>RoleMappingCacheSeconds</c> is only the fallback if that broadcast is lost.
|
||||
/// </summary>
|
||||
private static Task InvalidateRoleMappingCacheAfterCommit(IServiceProvider sp)
|
||||
{
|
||||
var cache = sp.GetService<RoleMappingCache>();
|
||||
return cache is null
|
||||
var peers = sp.GetService<ICentralCacheInvalidationPublisher>();
|
||||
return cache is null && peers is null
|
||||
? Task.CompletedTask
|
||||
: AfterConfigurationCommit(sp, () => { cache.Invalidate(); return Task.CompletedTask; });
|
||||
: AfterConfigurationCommit(sp, () =>
|
||||
{
|
||||
cache?.Invalidate();
|
||||
peers?.Publish(new RoleMappingsInvalidated());
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
private static async Task<object?> HandleCreateRoleMapping(IServiceProvider sp, CreateRoleMappingCommand cmd, string user)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Communication;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Security;
|
||||
|
||||
@@ -14,7 +16,10 @@ namespace ZB.MOM.WW.ScadaBridge.Security;
|
||||
/// an acceptable price: without an explicit <see cref="Invalidate"/>, a mapping or scope-rule
|
||||
/// change is visible on every node within that window. <see cref="Invalidate"/> discards this
|
||||
/// node's snapshot so the next mapping re-reads the database; calling it after a mutation makes
|
||||
/// the change immediate on the node that performed it (other nodes still rely on the TTL).
|
||||
/// the change immediate on the node that performed it. The peer central node is told over the
|
||||
/// central cache-invalidation topic (<see cref="RoleMappingsInvalidated"/>, audit D1,
|
||||
/// ScadaBridge#100 item 3), so a revoked mapping stops working there at once too; the TTL is
|
||||
/// only the fallback if that best-effort broadcast is lost.
|
||||
/// A TTL of <c>0</c> disables the cache: <see cref="Enabled"/> is false and
|
||||
/// <see cref="RoleMapper"/> reads the repository directly on every call.
|
||||
/// </summary>
|
||||
@@ -30,7 +35,7 @@ namespace ZB.MOM.WW.ScadaBridge.Security;
|
||||
/// read of the pre-change table can never overwrite the invalidation.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class RoleMappingCache
|
||||
public sealed class RoleMappingCache : ICentralCacheInvalidationHandler
|
||||
{
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IOptionsMonitor<SecurityOptions>? _options;
|
||||
@@ -65,6 +70,14 @@ public sealed class RoleMappingCache
|
||||
/// </summary>
|
||||
public bool Enabled => Ttl > TimeSpan.Zero;
|
||||
|
||||
/// <summary>Applies a peer central node's <see cref="RoleMappingsInvalidated"/>.</summary>
|
||||
/// <param name="invalidation">The received invalidation; other types are ignored.</param>
|
||||
public void Apply(ICentralCacheInvalidation invalidation)
|
||||
{
|
||||
if (invalidation is RoleMappingsInvalidated)
|
||||
Invalidate();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Discards the cached snapshot so the next mapping re-reads the database. Call after any
|
||||
/// LDAP-group-mapping or site-scope-rule create/update/delete.
|
||||
|
||||
@@ -65,6 +65,10 @@ public static class ServiceCollectionExtensions
|
||||
// (review 2026-10-21 05/P1). Singleton so every scoped RoleMapper shares it; the
|
||||
// repository it reads through is still the caller's scoped one.
|
||||
services.AddSingleton<RoleMappingCache>();
|
||||
// Audit D1 (ScadaBridge#100 item 3): a mapping change made through the other central node
|
||||
// reaches this node's snapshot over the central cache-invalidation topic.
|
||||
services.TryAddEnumerable(ServiceDescriptor.Singleton<ICentralCacheInvalidationHandler, RoleMappingCache>(
|
||||
sp => sp.GetRequiredService<RoleMappingCache>()));
|
||||
|
||||
// The cookie OnValidatePrincipal core. Scoped to match the
|
||||
// IGroupRoleMapper<string> it depends on (which depends on the Scoped
|
||||
|
||||
@@ -17,6 +17,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Scripting;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Scripting;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Transport;
|
||||
@@ -98,6 +99,9 @@ public sealed class BundleImporter : IBundleImporter
|
||||
// Optional (Gitea #83): drops this node's external-definition lookup cache after a
|
||||
// committed import. Null on hosts without the External System Gateway.
|
||||
private readonly IExternalDefinitionCacheInvalidator? _definitionCacheInvalidator;
|
||||
// Optional (ScadaBridge#100 item 3 / #83): tells the peer central node to drop its caches of
|
||||
// what the import wrote. Null on hosts without the central cache-invalidation bus.
|
||||
private readonly ICentralCacheInvalidationPublisher? _cacheInvalidationPublisher;
|
||||
private readonly ILogger<BundleImporter>? _logger;
|
||||
|
||||
/// <summary>
|
||||
@@ -125,6 +129,7 @@ public sealed class BundleImporter : IBundleImporter
|
||||
/// <param name="logger">Optional logger.</param>
|
||||
/// <param name="watermark">Optional: template-graph version watermark, bumped once per apply attempt so the flatten-session caches and the process-wide staleness memo cannot serve pre-import (or rolled-back) readings. Null on hosts that don't register it.</param>
|
||||
/// <param name="definitionCacheInvalidator">Optional (Gitea #83): drops this node's external-system / database-connection definition cache after a committed import, so imported definitions take effect immediately rather than after the cache TTL. Null on hosts without the External System Gateway.</param>
|
||||
/// <param name="cacheInvalidationPublisher">Optional (ScadaBridge#100 item 3): tells the peer central node to drop its caches of what a committed import wrote. Null on hosts without the central cache-invalidation bus.</param>
|
||||
public BundleImporter(
|
||||
BundleSerializer bundleSerializer,
|
||||
ManifestValidator manifestValidator,
|
||||
@@ -147,7 +152,8 @@ public sealed class BundleImporter : IBundleImporter
|
||||
IScriptArtifactChangeBus? scriptArtifactChangeBus = null,
|
||||
ILogger<BundleImporter>? logger = null,
|
||||
ITemplateGraphWatermark? watermark = null,
|
||||
IExternalDefinitionCacheInvalidator? definitionCacheInvalidator = null)
|
||||
IExternalDefinitionCacheInvalidator? definitionCacheInvalidator = null,
|
||||
ICentralCacheInvalidationPublisher? cacheInvalidationPublisher = null)
|
||||
{
|
||||
_bundleSerializer = bundleSerializer ?? throw new ArgumentNullException(nameof(bundleSerializer));
|
||||
_manifestValidator = manifestValidator ?? throw new ArgumentNullException(nameof(manifestValidator));
|
||||
@@ -170,6 +176,7 @@ public sealed class BundleImporter : IBundleImporter
|
||||
_scriptArtifactChangeBus = scriptArtifactChangeBus;
|
||||
_watermark = watermark;
|
||||
_definitionCacheInvalidator = definitionCacheInvalidator;
|
||||
_cacheInvalidationPublisher = cacheInvalidationPublisher;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -2498,18 +2505,20 @@ public sealed class BundleImporter : IBundleImporter
|
||||
/// served the pre-import definition for up to <c>DefinitionCacheTtl</c>. Invalidates on every
|
||||
/// committed import rather than only when an external entity was resolved: dependencies can
|
||||
/// pull definitions in implicitly, imports are rare, and a cleared cache only costs one
|
||||
/// re-query per name. Never throws (the transaction has already committed).
|
||||
/// re-query per name. The peer central node is told too (ScadaBridge#100 item 3), since its
|
||||
/// copy of the cache is just as stale. Never throws (the transaction has already committed).
|
||||
/// </summary>
|
||||
private void InvalidateExternalDefinitionCache()
|
||||
{
|
||||
if (_definitionCacheInvalidator is null)
|
||||
if (_definitionCacheInvalidator is null && _cacheInvalidationPublisher is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_definitionCacheInvalidator.InvalidateAll();
|
||||
_definitionCacheInvalidator?.InvalidateAll();
|
||||
_cacheInvalidationPublisher?.Publish(new ExternalDefinitionsInvalidated());
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user