using System.Collections.Immutable;
using Akka.Actor;
using Akka.Cluster.Tools.Client;
using Akka.Hosting;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
using ZB.MOM.WW.LocalDb;
namespace ZB.MOM.WW.OtOpcUa.Runtime;
public static class ServiceCollectionExtensions
{
public const string DriverRole = "driver";
public const string DriverHostActorName = "driver-host";
public const string DbHealthProbeActorName = "db-health";
public const string HistorianAdapterActorName = "historian-adapter";
public const string DependencyMuxActorName = "dependency-mux";
public const string OpcUaPublishActorName = "opcua-publish";
public const string PeerProbeSupervisorName = "peer-probe-supervisor";
public const string ContinuousHistorizationRecorderActorName = "continuous-historization-recorder";
public const string CentralClusterClientName = "central-cluster-client";
///
/// Registers shared runtime services. Currently binds
/// to as the default; production deployments
/// override this with LocalDbStoreAndForwardSink wrapping the HistorianGateway alarm writer.
/// Call this BEFORE AddAkka.
///
/// The service collection to register with.
/// The same instance for chaining.
public static IServiceCollection AddOtOpcUaRuntime(this IServiceCollection services)
{
services.TryAddSingleton(NullAlarmHistorianSink.Instance);
services.TryAddSingleton(NullHistorianDataSource.Instance);
// Historian tag provisioning. Null default (no-op) so the AddressSpaceApplier resolves a real
// IHistorianProvisioning only when AddHistorianProvisioning registers the gateway-backed one
// (ServerHistorian:Enabled). TryAddSingleton so the gateway registration wins last.
services.TryAddSingleton(NullHistorianProvisioning.Instance);
// VirtualTag historization sink. Null default — the durable AVEVA sink is infra-gated (there is
// no live-data historian write RPC). TryAddSingleton so a deployment that bound a real
// IHistoryWriter earlier wins.
services.TryAddSingleton(NullHistoryWriter.Instance);
services.TryAddSingleton(NullDriverFactory.Instance);
services.TryAddSingleton(NullOpcUaAddressSpaceSink.Instance);
services.TryAddSingleton(NullServiceLevelPublisher.Instance);
services.TryAddSingleton();
return services;
}
///
/// Config-gated durable alarm-historian sink. When the AlarmHistorian section has
/// Enabled=true, registers a (draining via the
/// -supplied writer) as the ,
/// overriding the default. Otherwise a no-op (Null stays).
/// The writer is injected so the durable downstream (the HistorianGateway alarm writer) can be
/// supplied by the Host, which is the only project that references it.
///
///
///
/// The queue lives in the node's consolidated , which replicates it
/// to the redundant pair peer — so undelivered alarm history survives losing the node
/// holding it. That is also why the drain is gated on :
/// the Secondary holds a full replica of the Primary's queue, and an ungated drain there
/// would re-deliver every event. Both services are resolved from the provider, so a
/// deployment that enables this section without registering LocalDb fails loudly at
/// resolution rather than silently buffering to nowhere.
///
///
/// The service collection to register with.
/// The configuration carrying the AlarmHistorian section.
///
/// Factory the Host supplies to build the concrete
/// (the HistorianGateway alarm writer) from the bound options + the resolving provider.
///
/// The same instance for chaining.
public static IServiceCollection AddAlarmHistorian(
this IServiceCollection services,
IConfiguration configuration,
Func writerFactory)
{
var opts = configuration.GetSection(AlarmHistorianOptions.SectionName).Get();
if (opts is not { Enabled: true }) return services; // leave the Null default from AddOtOpcUaRuntime
foreach (var warning in opts.Validate())
Serilog.Log.Logger.ForContext().Warning("Historian config: {HistorianConfigWarning}", warning);
// The view is a plain singleton so it exists even on nodes whose DriverHostActor never
// publishes to it — an unpublished view reads as "no peer, drain", which is the correct
// posture for a deployment that runs no redundancy at all.
services.TryAddSingleton();
// THE GATE ONLY APPLIES TO A REPLICATED QUEUE. Standing down is only ever safe because some
// other node holds the same rows and will send them instead — and the only node that does is
// this node's LocalDb replication peer. Without replication configured, these rows exist here
// and nowhere else, so deferring to anyone means they are never delivered by anyone.
//
// This is not hypothetical. The redundancy role is a CLUSTER-WIDE election
// (RedundancyStateActor keys on Akka's RoleLeader("driver")), while the queue is PAIR-LOCAL.
// On the docker-dev rig the elected driver Primary is a central node — which carries the
// driver Akka role, replicates nobody's LocalDb, and does not even run the alarm historian —
// so every site node dutifully suspended its drain in favour of a node that could not
// possibly deliver its events. Scoping the gate to "is my queue actually shared?" is what
// keeps the two scopes from disagreeing.
// BOTH halves of a pair share the queue, but only one of them dials: the initiator sets
// Replication:PeerAddress, its partner only sets SyncListenPort and waits. Testing the dial
// side alone would leave the listening half permanently ungated — one drainer per pair by
// accident rather than by role, and the wrong one whenever the roles swap.
var replicated =
!string.IsNullOrWhiteSpace(configuration["LocalDb:Replication:PeerAddress"])
|| !string.IsNullOrWhiteSpace(configuration["LocalDb:SyncListenPort"]);
if (!replicated)
{
Serilog.Log.Logger.ForContext().Information(
"Alarm historian: LocalDb replication is not configured, so this node's queue is not "
+ "shared with any peer and the Primary drain gate does not apply — this node always "
+ "drains its own alarm queue.");
}
services.AddSingleton(sp =>
{
// LocalDbStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging).
// Resolve it off the host's configured static logger so the drain worker's WARN/INFO
// lines land in the same sinks as the rest of the process.
var roleView = sp.GetRequiredService();
var sink = new LocalDbStoreAndForwardSink(
sp.GetRequiredService(),
writerFactory(opts, sp),
Serilog.Log.Logger.ForContext(),
batchSize: opts.BatchSize,
capacity: opts.Capacity,
deadLetterRetention: TimeSpan.FromDays(opts.DeadLetterRetentionDays),
maxAttempts: opts.MaxAttempts,
drainGate: () => !replicated || roleView.ShouldDrainAlarmHistory);
sink.StartDrainLoop(TimeSpan.FromSeconds(opts.DrainIntervalSeconds));
return sink;
});
return services;
}
///
/// Config-gated server-side HistoryRead backend. When the ServerHistorian section has
/// Enabled=true, registers the -supplied
/// (the read-only HistorianGateway-backed data source) overriding
/// the default from . Otherwise
/// a no-op (the Null default stays and the node manager's HistoryRead returns
/// GoodNoData-empty). The data source is injected so the gateway-backed client can be supplied
/// by the Host, which is the only project that references the driver.
///
/// The service collection to register with.
/// The configuration carrying the ServerHistorian section.
///
/// Factory the Host supplies to build the concrete read
/// (the gateway-backed data source) from the bound options + the resolving provider.
///
/// The same instance for chaining.
public static IServiceCollection AddServerHistorian(
this IServiceCollection services,
IConfiguration configuration,
Func dataSourceFactory)
{
var opts = configuration.GetSection(ServerHistorianOptions.SectionName).Get();
if (opts is not { Enabled: true }) return services; // leave the Null default from AddOtOpcUaRuntime
foreach (var warning in opts.Validate())
Serilog.Log.Logger.ForContext().Warning("ServerHistorian config: {ServerHistorianConfigWarning}", warning);
// Last-registration-wins over the TryAddSingleton Null default seeded by AddOtOpcUaRuntime.
services.AddSingleton(sp => dataSourceFactory(opts, sp));
return services;
}
///
/// Config-gated historian tag provisioning. When the ServerHistorian section has
/// Enabled=true, registers the -supplied
/// (the gateway-backed GatewayTagProvisioner that calls
/// the gateway's EnsureTags) overriding the default from
/// . Otherwise a no-op (the Null default stays and deploying historized
/// tags provisions nothing). The provisioner is consumed by the AddressSpaceApplier, which fires
/// a non-blocking for added historized tags on
/// every deploy. Gated on the same ServerHistorian:Enabled flag as the read path
/// () — provisioning targets the same single gateway. The provisioner
/// is injected so the gateway-backed client can be supplied by the Host, which is the only project that
/// references the driver.
///
/// The service collection to register with.
/// The configuration carrying the ServerHistorian section.
///
/// Factory the Host supplies to build the concrete
/// (the gateway-backed provisioner) from the bound options + the resolving provider.
///
/// The same instance for chaining.
public static IServiceCollection AddHistorianProvisioning(
this IServiceCollection services,
IConfiguration configuration,
Func provisioningFactory)
{
var opts = configuration.GetSection(ServerHistorianOptions.SectionName).Get();
if (opts is not { Enabled: true }) return services; // leave the Null default from AddOtOpcUaRuntime
// Last-registration-wins over the TryAddSingleton Null default seeded by AddOtOpcUaRuntime.
services.AddSingleton(sp => provisioningFactory(opts, sp));
return services;
}
///
/// Spawns the per-node driver-role actors on the host's :
/// (one per node),
/// (consumed by the health endpoint + redundancy calc), and
/// wrapping the registered .
///
/// Mirror of WithOtOpcUaControlPlaneSingletons for the driver role. Both must
/// be registered on the same as the cluster
/// bootstrap so the actors share the host's ActorSystem.
///
/// Wire from the fused Host's Program.cs when the node carries the driver role:
///
/// services.AddOtOpcUaRuntime();
/// services.AddAkka("otopcua", (ab, sp) => { ab.WithOtOpcUaClusterBootstrap(sp); if (hasDriver) ab.WithOtOpcUaRuntimeActors(); });
///
///
/// The Akka configuration builder.
/// The same instance for chaining.
public static AkkaConfigurationBuilder WithOtOpcUaRuntimeActors(this AkkaConfigurationBuilder builder)
{
// Production cluster HOCON (akka.conf) carries this dispatcher block, but consumers that
// bootstrap their own HOCON (e.g. ServiceCollectionExtensionsTests) wouldn't pick it up
// — OpcUaPublishActor.Props pins itself to opcua-synchronized-dispatcher and Akka throws
// ConfigurationException if it doesn't exist. Prepend a fallback so the runtime extension
// is self-contained.
builder.AddHocon(@"
opcua-synchronized-dispatcher {
type = ""PinnedDispatcher""
executor = ""thread-pool-executor""
throughput = 1
}
", HoconAddMode.Prepend);
builder.WithActors((system, registry, resolver) =>
{
var dbFactory = resolver.GetService>();
var roleInfo = resolver.GetService();
// Fallback to Null* if AddOtOpcUaRuntime wasn't called (e.g., test harnesses).
var historianSink = resolver.GetService() ?? NullAlarmHistorianSink.Instance;
var driverFactory = resolver.GetService() ?? NullDriverFactory.Instance;
// Phase 6.1 resilience-invoker factory — bound by the fused Host (which references Core +
// Polly); pass-through on nodes/harnesses that didn't register it. Resolved through the
// Core.Abstractions seam so this Polly-free assembly stays that way.
var invokerFactory = resolver.GetService()
?? NullDriverCapabilityInvokerFactory.Instance;
var addressSpaceSink = resolver.GetService() ?? NullOpcUaAddressSpaceSink.Instance;
var serviceLevel = resolver.GetService() ?? NullServiceLevelPublisher.Instance;
var loggerFactory = resolver.GetService() ?? NullLoggerFactory.Instance;
var healthPublisher = resolver.GetService() ?? NullDriverHealthPublisher.Instance;
// Node-local deployment-artifact cache. Registered by the Host's AddOtOpcUaLocalDb on
// driver-role nodes only; deliberately left null elsewhere (admin-only graphs, test
// harnesses) rather than given a null-object, so DriverHostActor skips caching outright
// instead of pretending to cache into a sink that drops everything.
var deploymentArtifactCache = resolver.GetService();
// Per-cluster mesh Phase 3: ConfigSource:Mode selects where this driver reads config.
// FetchAndCache pulls the artifact from central over gRPC and reads only the LocalDb cache;
// Direct (default) reads central SQL as before. The fetcher is resolved only in FetchAndCache
// mode (registered by the Host under hasDriver); its absence there is a misconfiguration the
// actor fails the apply on rather than fetching nothing forever.
var configSourceOptions =
resolver.GetService>()?.Value ?? new ConfigSourceOptions();
var fetchAndCacheMode = string.Equals(
configSourceOptions.Mode, ConfigSourceOptions.ModeFetchAndCache, StringComparison.OrdinalIgnoreCase);
var artifactFetcher = fetchAndCacheMode ? resolver.GetService() : null;
// Where this actor publishes its Primary-gate verdict for the alarm store-and-forward
// drain, which runs on a timer and so cannot read RedundancyStateChanged itself.
// Registered by AddAlarmHistorian; absent when no durable sink is configured, in which
// case there is nothing downstream to inform.
var redundancyRoleView = resolver.GetService();
// Host of this node's LocalDb replication partner: the ONLY node that holds a copy of this
// node's alarm queue, and so the only node it may stand down in favour of. Null when this
// node dials nobody, which correctly means "never stand down".
var replicationPeerHost =
Uri.TryCreate(
resolver.GetService()?["LocalDb:Replication:PeerAddress"],
UriKind.Absolute,
out var peerUri)
? peerUri.Host
: null;
// Root script logger backs the ScriptedAlarm host's engine + script logging. Registered in
// Host DI inside the hasDriver block; may be absent in some role configs / test harnesses,
// in which case the DriverHostActor gracefully skips spawning the ScriptedAlarm host.
var scriptRootLogger = resolver.GetService();
// Production evaluator is the Host's RoslynVirtualTagEvaluator (registered as
// IVirtualTagEvaluator); fall back to the null evaluator for test harnesses that don't
// register one (VirtualTagActor children then evaluate to nothing).
var virtualTagEvaluator = resolver.GetService();
if (virtualTagEvaluator is null)
{
loggerFactory.CreateLogger("ZB.MOM.WW.OtOpcUa.Runtime.ServiceCollectionExtensions")
.LogWarning("IVirtualTagEvaluator not registered; Equipment VirtualTags will evaluate to NoChange (no live values). Expected only in test harnesses — driver-role nodes should register RoslynVirtualTagEvaluator.");
virtualTagEvaluator = NullVirtualTagEvaluator.Instance;
}
// VirtualTag historization sink threaded to the spawned VirtualTagHostActor. Null default
// (durable AVEVA sink is infra-gated); a deployment binding a real IHistoryWriter overrides.
var historyWriter = resolver.GetService() ?? NullHistoryWriter.Instance;
var dbHealth = system.ActorOf(
DbHealthProbeActor.Props(dbFactory),
DbHealthProbeActorName);
registry.Register(dbHealth);
// Dependency mux must be spawned before DriverHostActor so the host can forward
// AttributeValuePublished into it from the very first driver spawn.
var mux = system.ActorOf(DependencyMuxActor.Props(), DependencyMuxActorName);
registry.Register(mux);
// Per-cluster mesh Phase 2: the node end of the central↔node command boundary. Spawned
// BEFORE DriverHostActor because it is the host's ackRouter under ClusterClient mode.
// It has no dependency of its own on the host — inbound commands travel via the node's
// EventStream — so this ordering costs nothing.
var meshOptions = resolver.GetService>().Value;
var useClusterClient = string.Equals(
meshOptions.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase);
IActorRef? centralClient = null;
if (useClusterClient)
{
// Contacts come from appsettings, not the database — the deliberate asymmetry:
// central's node set changes as operators add and retire nodes, but central's own
// address is part of the deployment. MeshTransportOptionsValidator has already
// rejected an empty or malformed list at boot, so this cannot silently produce a
// client with no contacts.
var contacts = meshOptions.CentralContactPoints
.Select(cp => ActorPath.Parse($"{cp}/system/receptionist"))
.ToImmutableHashSet();
centralClient = system.ActorOf(
ClusterClient.Props(ClusterClientSettings.Create(system).WithInitialContacts(contacts)),
CentralClusterClientName);
}
var nodeComm = system.ActorOf(
NodeCommunicationActor.Props(centralClient),
MeshPaths.NodeCommunicationName);
// Registered with the receptionist in BOTH modes: under Dps nothing dials it, and an
// idle registration costs nothing — whereas registering only under ClusterClient would
// make flipping the flag require a node restart before central could reach it.
ClusterClientReceptionist.Get(system).RegisterService(nodeComm);
registry.Register(nodeComm);
// Continuous-historization recorder — gated on ContinuousHistorization:Enabled AND the
// gateway-backed IHistorianValueWriter + the durable IHistorizationOutbox being registered
// (the Host registers both ONLY when historization is enabled and the ServerHistorian gateway
// is configured). The recorder taps the dependency mux's value fan-out, so it is spawned after
// (and fed) the same `mux` ref the DriverHostActor uses. It is spawned BEFORE the applier so
// the applier's historized-ref subscription sink can wrap this recorder's IActorRef and feed it
// the add/remove delta of historized refs on every deploy (closing the T18 ref-feed gap).
IActorRef? continuousRecorder = null;
var continuousOptions = resolver.GetService();
if (continuousOptions is { Enabled: true })
{
var valueWriter = resolver.GetService();
var outbox = resolver.GetService();
if (valueWriter is not null && outbox is not null)
{
// Initial ref set is EMPTY: the deployed address space (and thus the historized-ref
// set) is built later at deploy time, not here. The applier's per-deploy add/remove
// feed populates the recorder's interest from that point on.
continuousRecorder = system.ActorOf(
ContinuousHistorizationRecorder.Props(
dependencyMux: mux,
writer: valueWriter,
outbox: outbox,
historizedRefs: Array.Empty(),
drainBatchSize: continuousOptions.DrainBatchSize,
drainInterval: TimeSpan.FromSeconds(continuousOptions.DrainIntervalSeconds),
minBackoff: TimeSpan.FromSeconds(continuousOptions.MinBackoffSeconds),
maxBackoff: TimeSpan.FromSeconds(continuousOptions.MaxBackoffSeconds)),
ContinuousHistorizationRecorderActorName);
registry.Register(continuousRecorder);
}
else
{
loggerFactory.CreateLogger("ZB.MOM.WW.OtOpcUa.Runtime.ServiceCollectionExtensions")
.LogWarning("ContinuousHistorization is enabled but IHistorianValueWriter and/or IHistorizationOutbox are not registered; the recorder will not be spawned. Expected only in misconfigured deployments or test harnesses.");
}
}
// Historized-ref subscription sink fed by the applier on every deploy. When the recorder was
// spawned, an adapter wraps its IActorRef (a non-blocking Tell of the add/remove delta);
// otherwise the Null no-op sink, so the applier behaves identically when historization is off.
IHistorizedTagSubscriptionSink historizedSubscriptions = continuousRecorder is not null
? new ActorHistorizedTagSubscriptionSink(continuousRecorder)
: NullHistorizedTagSubscriptionSink.Instance;
// Historian tag provisioner fed to the applier so deploying historized tags auto-ensures
// them in the historian (EnsureTags). The Host registers the gateway-backed provisioner via
// AddHistorianProvisioning when ServerHistorian:Enabled; otherwise this resolves the no-op
// NullHistorianProvisioning default seeded by AddOtOpcUaRuntime (so the applier's hook is inert).
var provisioning = resolver.GetService();
// OPC UA publish actor — pinned dispatcher, owns the address-space side of the
// pipeline. AddressSpaceApplier is constructed here so the actor + applier share the
// same sink reference (when DeferredAddressSpaceSink swaps later, both see it).
var applier = new AddressSpaceApplier(
addressSpaceSink,
loggerFactory.CreateLogger(),
provisioning: provisioning,
historizedSubscriptions: historizedSubscriptions);
var publishActor = system.ActorOf(
OpcUaPublishActor.Props(
sink: addressSpaceSink,
serviceLevel: serviceLevel,
localNode: roleInfo.LocalNode,
dbFactory: dbFactory,
applier: applier,
dbHealthProbe: dbHealth),
OpcUaPublishActorName);
registry.Register(publishActor);
// Per-node peer-probe supervisor — keeps one OPC UA TCP probe per OTHER non-Detached
// driver node, so every node is continuously probed by all its peers. OpcUaPublishActor
// consumes the resulting probe verdicts to learn this node's own reachability.
var peerProbes = system.ActorOf(
PeerProbeSupervisor.Props(roleInfo.LocalNode),
PeerProbeSupervisorName);
registry.Register(peerProbes);
var driverHost = system.ActorOf(
// ackRouter: under ClusterClient mode the node comm actor relays ApplyAcks across
// the boundary; under Dps it stays null and the host publishes on the
// deployment-acks topic exactly as before.
DriverHostActor.Props(dbFactory, roleInfo.LocalNode,
ackRouter: useClusterClient ? nodeComm : null,
driverFactory: driverFactory, localRoles: roleInfo.LocalRoles,
dependencyMux: mux,
opcUaPublishActor: publishActor,
healthPublisher: healthPublisher,
virtualTagEvaluator: virtualTagEvaluator,
historyWriter: historyWriter,
loggerFactory: loggerFactory,
scriptRootLogger: scriptRootLogger,
invokerFactory: invokerFactory,
deploymentArtifactCache: deploymentArtifactCache,
redundancyRoleView: redundancyRoleView,
replicationPeerHost: replicationPeerHost,
fetchAndCacheMode: fetchAndCacheMode,
artifactFetcher: artifactFetcher),
DriverHostActorName);
registry.Register(driverHost);
var historian = system.ActorOf(
HistorianAdapterActor.Props(historianSink, roleInfo.LocalNode),
HistorianAdapterActorName);
registry.Register(historian);
});
return builder;
}
}
/// Marker key types used by Akka.Hosting to resolve runtime actors from the registry.
public sealed class DriverHostActorKey { }
public sealed class DbHealthProbeActorKey { }
public sealed class HistorianAdapterActorKey { }
public sealed class DependencyMuxActorKey { }
public sealed class OpcUaPublishActorKey { }
public sealed class NodeCommunicationActorKey { }
/// Marker key for the per-node ContinuousHistorizationRecorder (spawned only when
/// ContinuousHistorization:Enabled=true and the gateway value-writer + outbox are registered).
public sealed class ContinuousHistorizationRecorderKey { }
/// Marker key for the per-node PeerProbeSupervisor.
public sealed class PeerProbeSupervisorKey { }