using System.Collections.Immutable; using Akka.Actor; using Akka.Cluster.Tools.Client; using Akka.Hosting; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.Runtime.Communication; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Core.VirtualTags; using ZB.MOM.WW.OtOpcUa.OpcUaServer; using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache; using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.Health; using ZB.MOM.WW.OtOpcUa.Runtime.Historian; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; using ZB.MOM.WW.LocalDb; namespace ZB.MOM.WW.OtOpcUa.Runtime; public static class ServiceCollectionExtensions { public const string DriverRole = "driver"; public const string DriverHostActorName = "driver-host"; public const string DbHealthProbeActorName = "db-health"; public const string HistorianAdapterActorName = "historian-adapter"; public const string DependencyMuxActorName = "dependency-mux"; public const string OpcUaPublishActorName = "opcua-publish"; public const string PeerProbeSupervisorName = "peer-probe-supervisor"; public const string ContinuousHistorizationRecorderActorName = "continuous-historization-recorder"; public const string CentralClusterClientName = "central-cluster-client"; /// /// Registers shared runtime services. Currently binds /// to as the default; production deployments /// override this with LocalDbStoreAndForwardSink wrapping the HistorianGateway alarm writer. /// Call this BEFORE AddAkka. /// /// The service collection to register with. /// The same instance for chaining. public static IServiceCollection AddOtOpcUaRuntime(this IServiceCollection services) { services.TryAddSingleton(NullAlarmHistorianSink.Instance); services.TryAddSingleton(NullHistorianDataSource.Instance); // Historian tag provisioning. Null default (no-op) so the AddressSpaceApplier resolves a real // IHistorianProvisioning only when AddHistorianProvisioning registers the gateway-backed one // (ServerHistorian:Enabled). TryAddSingleton so the gateway registration wins last. services.TryAddSingleton(NullHistorianProvisioning.Instance); // VirtualTag historization sink. Null default — the durable AVEVA sink is infra-gated (there is // no live-data historian write RPC). TryAddSingleton so a deployment that bound a real // IHistoryWriter earlier wins. services.TryAddSingleton(NullHistoryWriter.Instance); services.TryAddSingleton(NullDriverFactory.Instance); services.TryAddSingleton(NullOpcUaAddressSpaceSink.Instance); services.TryAddSingleton(NullServiceLevelPublisher.Instance); services.TryAddSingleton(); return services; } /// /// Config-gated durable alarm-historian sink. When the AlarmHistorian section has /// Enabled=true, registers a (draining via the /// -supplied writer) as the , /// overriding the default. Otherwise a no-op (Null stays). /// The writer is injected so the durable downstream (the HistorianGateway alarm writer) can be /// supplied by the Host, which is the only project that references it. /// /// /// /// The queue lives in the node's consolidated , which replicates it /// to the redundant pair peer — so undelivered alarm history survives losing the node /// holding it. That is also why the drain is gated on : /// the Secondary holds a full replica of the Primary's queue, and an ungated drain there /// would re-deliver every event. Both services are resolved from the provider, so a /// deployment that enables this section without registering LocalDb fails loudly at /// resolution rather than silently buffering to nowhere. /// /// /// The service collection to register with. /// The configuration carrying the AlarmHistorian section. /// /// Factory the Host supplies to build the concrete /// (the HistorianGateway alarm writer) from the bound options + the resolving provider. /// /// The same instance for chaining. public static IServiceCollection AddAlarmHistorian( this IServiceCollection services, IConfiguration configuration, Func writerFactory) { var opts = configuration.GetSection(AlarmHistorianOptions.SectionName).Get(); if (opts is not { Enabled: true }) return services; // leave the Null default from AddOtOpcUaRuntime foreach (var warning in opts.Validate()) Serilog.Log.Logger.ForContext().Warning("Historian config: {HistorianConfigWarning}", warning); // The view is a plain singleton so it exists even on nodes whose DriverHostActor never // publishes to it — an unpublished view reads as "no peer, drain", which is the correct // posture for a deployment that runs no redundancy at all. services.TryAddSingleton(); // THE GATE ONLY APPLIES TO A REPLICATED QUEUE. Standing down is only ever safe because some // other node holds the same rows and will send them instead — and the only node that does is // this node's LocalDb replication peer. Without replication configured, these rows exist here // and nowhere else, so deferring to anyone means they are never delivered by anyone. // // This is not hypothetical. The redundancy role is a CLUSTER-WIDE election // (RedundancyStateActor keys on Akka's RoleLeader("driver")), while the queue is PAIR-LOCAL. // On the docker-dev rig the elected driver Primary is a central node — which carries the // driver Akka role, replicates nobody's LocalDb, and does not even run the alarm historian — // so every site node dutifully suspended its drain in favour of a node that could not // possibly deliver its events. Scoping the gate to "is my queue actually shared?" is what // keeps the two scopes from disagreeing. // BOTH halves of a pair share the queue, but only one of them dials: the initiator sets // Replication:PeerAddress, its partner only sets SyncListenPort and waits. Testing the dial // side alone would leave the listening half permanently ungated — one drainer per pair by // accident rather than by role, and the wrong one whenever the roles swap. var replicated = !string.IsNullOrWhiteSpace(configuration["LocalDb:Replication:PeerAddress"]) || !string.IsNullOrWhiteSpace(configuration["LocalDb:SyncListenPort"]); if (!replicated) { Serilog.Log.Logger.ForContext().Information( "Alarm historian: LocalDb replication is not configured, so this node's queue is not " + "shared with any peer and the Primary drain gate does not apply — this node always " + "drains its own alarm queue."); } services.AddSingleton(sp => { // LocalDbStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging). // Resolve it off the host's configured static logger so the drain worker's WARN/INFO // lines land in the same sinks as the rest of the process. var roleView = sp.GetRequiredService(); var sink = new LocalDbStoreAndForwardSink( sp.GetRequiredService(), writerFactory(opts, sp), Serilog.Log.Logger.ForContext(), batchSize: opts.BatchSize, capacity: opts.Capacity, deadLetterRetention: TimeSpan.FromDays(opts.DeadLetterRetentionDays), maxAttempts: opts.MaxAttempts, drainGate: () => !replicated || roleView.ShouldDrainAlarmHistory); sink.StartDrainLoop(TimeSpan.FromSeconds(opts.DrainIntervalSeconds)); return sink; }); return services; } /// /// Config-gated server-side HistoryRead backend. When the ServerHistorian section has /// Enabled=true, registers the -supplied /// (the read-only HistorianGateway-backed data source) overriding /// the default from . Otherwise /// a no-op (the Null default stays and the node manager's HistoryRead returns /// GoodNoData-empty). The data source is injected so the gateway-backed client can be supplied /// by the Host, which is the only project that references the driver. /// /// The service collection to register with. /// The configuration carrying the ServerHistorian section. /// /// Factory the Host supplies to build the concrete read /// (the gateway-backed data source) from the bound options + the resolving provider. /// /// The same instance for chaining. public static IServiceCollection AddServerHistorian( this IServiceCollection services, IConfiguration configuration, Func dataSourceFactory) { var opts = configuration.GetSection(ServerHistorianOptions.SectionName).Get(); if (opts is not { Enabled: true }) return services; // leave the Null default from AddOtOpcUaRuntime foreach (var warning in opts.Validate()) Serilog.Log.Logger.ForContext().Warning("ServerHistorian config: {ServerHistorianConfigWarning}", warning); // Last-registration-wins over the TryAddSingleton Null default seeded by AddOtOpcUaRuntime. services.AddSingleton(sp => dataSourceFactory(opts, sp)); return services; } /// /// Config-gated historian tag provisioning. When the ServerHistorian section has /// Enabled=true, registers the -supplied /// (the gateway-backed GatewayTagProvisioner that calls /// the gateway's EnsureTags) overriding the default from /// . Otherwise a no-op (the Null default stays and deploying historized /// tags provisions nothing). The provisioner is consumed by the AddressSpaceApplier, which fires /// a non-blocking for added historized tags on /// every deploy. Gated on the same ServerHistorian:Enabled flag as the read path /// () — provisioning targets the same single gateway. The provisioner /// is injected so the gateway-backed client can be supplied by the Host, which is the only project that /// references the driver. /// /// The service collection to register with. /// The configuration carrying the ServerHistorian section. /// /// Factory the Host supplies to build the concrete /// (the gateway-backed provisioner) from the bound options + the resolving provider. /// /// The same instance for chaining. public static IServiceCollection AddHistorianProvisioning( this IServiceCollection services, IConfiguration configuration, Func provisioningFactory) { var opts = configuration.GetSection(ServerHistorianOptions.SectionName).Get(); if (opts is not { Enabled: true }) return services; // leave the Null default from AddOtOpcUaRuntime // Last-registration-wins over the TryAddSingleton Null default seeded by AddOtOpcUaRuntime. services.AddSingleton(sp => provisioningFactory(opts, sp)); return services; } /// /// Spawns the per-node driver-role actors on the host's : /// (one per node), /// (consumed by the health endpoint + redundancy calc), and /// wrapping the registered . /// /// Mirror of WithOtOpcUaControlPlaneSingletons for the driver role. Both must /// be registered on the same as the cluster /// bootstrap so the actors share the host's ActorSystem. /// /// Wire from the fused Host's Program.cs when the node carries the driver role: /// /// services.AddOtOpcUaRuntime(); /// services.AddAkka("otopcua", (ab, sp) => { ab.WithOtOpcUaClusterBootstrap(sp); if (hasDriver) ab.WithOtOpcUaRuntimeActors(); }); /// /// /// The Akka configuration builder. /// The same instance for chaining. public static AkkaConfigurationBuilder WithOtOpcUaRuntimeActors(this AkkaConfigurationBuilder builder) { // Production cluster HOCON (akka.conf) carries this dispatcher block, but consumers that // bootstrap their own HOCON (e.g. ServiceCollectionExtensionsTests) wouldn't pick it up // — OpcUaPublishActor.Props pins itself to opcua-synchronized-dispatcher and Akka throws // ConfigurationException if it doesn't exist. Prepend a fallback so the runtime extension // is self-contained. builder.AddHocon(@" opcua-synchronized-dispatcher { type = ""PinnedDispatcher"" executor = ""thread-pool-executor"" throughput = 1 } ", HoconAddMode.Prepend); builder.WithActors((system, registry, resolver) => { var dbFactory = resolver.GetService>(); var roleInfo = resolver.GetService(); // Fallback to Null* if AddOtOpcUaRuntime wasn't called (e.g., test harnesses). var historianSink = resolver.GetService() ?? NullAlarmHistorianSink.Instance; var driverFactory = resolver.GetService() ?? NullDriverFactory.Instance; // Phase 6.1 resilience-invoker factory — bound by the fused Host (which references Core + // Polly); pass-through on nodes/harnesses that didn't register it. Resolved through the // Core.Abstractions seam so this Polly-free assembly stays that way. var invokerFactory = resolver.GetService() ?? NullDriverCapabilityInvokerFactory.Instance; var addressSpaceSink = resolver.GetService() ?? NullOpcUaAddressSpaceSink.Instance; var serviceLevel = resolver.GetService() ?? NullServiceLevelPublisher.Instance; var loggerFactory = resolver.GetService() ?? NullLoggerFactory.Instance; var healthPublisher = resolver.GetService() ?? NullDriverHealthPublisher.Instance; // Node-local deployment-artifact cache. Registered by the Host's AddOtOpcUaLocalDb on // driver-role nodes only; deliberately left null elsewhere (admin-only graphs, test // harnesses) rather than given a null-object, so DriverHostActor skips caching outright // instead of pretending to cache into a sink that drops everything. var deploymentArtifactCache = resolver.GetService(); // Per-cluster mesh Phase 3: ConfigSource:Mode selects where this driver reads config. // FetchAndCache pulls the artifact from central over gRPC and reads only the LocalDb cache; // Direct (default) reads central SQL as before. The fetcher is resolved only in FetchAndCache // mode (registered by the Host under hasDriver); its absence there is a misconfiguration the // actor fails the apply on rather than fetching nothing forever. var configSourceOptions = resolver.GetService>()?.Value ?? new ConfigSourceOptions(); var fetchAndCacheMode = string.Equals( configSourceOptions.Mode, ConfigSourceOptions.ModeFetchAndCache, StringComparison.OrdinalIgnoreCase); var artifactFetcher = fetchAndCacheMode ? resolver.GetService() : null; // Where this actor publishes its Primary-gate verdict for the alarm store-and-forward // drain, which runs on a timer and so cannot read RedundancyStateChanged itself. // Registered by AddAlarmHistorian; absent when no durable sink is configured, in which // case there is nothing downstream to inform. var redundancyRoleView = resolver.GetService(); // Host of this node's LocalDb replication partner: the ONLY node that holds a copy of this // node's alarm queue, and so the only node it may stand down in favour of. Null when this // node dials nobody, which correctly means "never stand down". var replicationPeerHost = Uri.TryCreate( resolver.GetService()?["LocalDb:Replication:PeerAddress"], UriKind.Absolute, out var peerUri) ? peerUri.Host : null; // Root script logger backs the ScriptedAlarm host's engine + script logging. Registered in // Host DI inside the hasDriver block; may be absent in some role configs / test harnesses, // in which case the DriverHostActor gracefully skips spawning the ScriptedAlarm host. var scriptRootLogger = resolver.GetService(); // Production evaluator is the Host's RoslynVirtualTagEvaluator (registered as // IVirtualTagEvaluator); fall back to the null evaluator for test harnesses that don't // register one (VirtualTagActor children then evaluate to nothing). var virtualTagEvaluator = resolver.GetService(); if (virtualTagEvaluator is null) { loggerFactory.CreateLogger("ZB.MOM.WW.OtOpcUa.Runtime.ServiceCollectionExtensions") .LogWarning("IVirtualTagEvaluator not registered; Equipment VirtualTags will evaluate to NoChange (no live values). Expected only in test harnesses — driver-role nodes should register RoslynVirtualTagEvaluator."); virtualTagEvaluator = NullVirtualTagEvaluator.Instance; } // VirtualTag historization sink threaded to the spawned VirtualTagHostActor. Null default // (durable AVEVA sink is infra-gated); a deployment binding a real IHistoryWriter overrides. var historyWriter = resolver.GetService() ?? NullHistoryWriter.Instance; var dbHealth = system.ActorOf( DbHealthProbeActor.Props(dbFactory), DbHealthProbeActorName); registry.Register(dbHealth); // Dependency mux must be spawned before DriverHostActor so the host can forward // AttributeValuePublished into it from the very first driver spawn. var mux = system.ActorOf(DependencyMuxActor.Props(), DependencyMuxActorName); registry.Register(mux); // Per-cluster mesh Phase 2: the node end of the central↔node command boundary. Spawned // BEFORE DriverHostActor because it is the host's ackRouter under ClusterClient mode. // It has no dependency of its own on the host — inbound commands travel via the node's // EventStream — so this ordering costs nothing. var meshOptions = resolver.GetService>().Value; var useClusterClient = string.Equals( meshOptions.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase); IActorRef? centralClient = null; if (useClusterClient) { // Contacts come from appsettings, not the database — the deliberate asymmetry: // central's node set changes as operators add and retire nodes, but central's own // address is part of the deployment. MeshTransportOptionsValidator has already // rejected an empty or malformed list at boot, so this cannot silently produce a // client with no contacts. var contacts = meshOptions.CentralContactPoints .Select(cp => ActorPath.Parse($"{cp}/system/receptionist")) .ToImmutableHashSet(); centralClient = system.ActorOf( ClusterClient.Props(ClusterClientSettings.Create(system).WithInitialContacts(contacts)), CentralClusterClientName); } var nodeComm = system.ActorOf( NodeCommunicationActor.Props(centralClient), MeshPaths.NodeCommunicationName); // Registered with the receptionist in BOTH modes: under Dps nothing dials it, and an // idle registration costs nothing — whereas registering only under ClusterClient would // make flipping the flag require a node restart before central could reach it. ClusterClientReceptionist.Get(system).RegisterService(nodeComm); registry.Register(nodeComm); // Continuous-historization recorder — gated on ContinuousHistorization:Enabled AND the // gateway-backed IHistorianValueWriter + the durable IHistorizationOutbox being registered // (the Host registers both ONLY when historization is enabled and the ServerHistorian gateway // is configured). The recorder taps the dependency mux's value fan-out, so it is spawned after // (and fed) the same `mux` ref the DriverHostActor uses. It is spawned BEFORE the applier so // the applier's historized-ref subscription sink can wrap this recorder's IActorRef and feed it // the add/remove delta of historized refs on every deploy (closing the T18 ref-feed gap). IActorRef? continuousRecorder = null; var continuousOptions = resolver.GetService(); if (continuousOptions is { Enabled: true }) { var valueWriter = resolver.GetService(); var outbox = resolver.GetService(); if (valueWriter is not null && outbox is not null) { // Initial ref set is EMPTY: the deployed address space (and thus the historized-ref // set) is built later at deploy time, not here. The applier's per-deploy add/remove // feed populates the recorder's interest from that point on. continuousRecorder = system.ActorOf( ContinuousHistorizationRecorder.Props( dependencyMux: mux, writer: valueWriter, outbox: outbox, historizedRefs: Array.Empty(), drainBatchSize: continuousOptions.DrainBatchSize, drainInterval: TimeSpan.FromSeconds(continuousOptions.DrainIntervalSeconds), minBackoff: TimeSpan.FromSeconds(continuousOptions.MinBackoffSeconds), maxBackoff: TimeSpan.FromSeconds(continuousOptions.MaxBackoffSeconds)), ContinuousHistorizationRecorderActorName); registry.Register(continuousRecorder); } else { loggerFactory.CreateLogger("ZB.MOM.WW.OtOpcUa.Runtime.ServiceCollectionExtensions") .LogWarning("ContinuousHistorization is enabled but IHistorianValueWriter and/or IHistorizationOutbox are not registered; the recorder will not be spawned. Expected only in misconfigured deployments or test harnesses."); } } // Historized-ref subscription sink fed by the applier on every deploy. When the recorder was // spawned, an adapter wraps its IActorRef (a non-blocking Tell of the add/remove delta); // otherwise the Null no-op sink, so the applier behaves identically when historization is off. IHistorizedTagSubscriptionSink historizedSubscriptions = continuousRecorder is not null ? new ActorHistorizedTagSubscriptionSink(continuousRecorder) : NullHistorizedTagSubscriptionSink.Instance; // Historian tag provisioner fed to the applier so deploying historized tags auto-ensures // them in the historian (EnsureTags). The Host registers the gateway-backed provisioner via // AddHistorianProvisioning when ServerHistorian:Enabled; otherwise this resolves the no-op // NullHistorianProvisioning default seeded by AddOtOpcUaRuntime (so the applier's hook is inert). var provisioning = resolver.GetService(); // OPC UA publish actor — pinned dispatcher, owns the address-space side of the // pipeline. AddressSpaceApplier is constructed here so the actor + applier share the // same sink reference (when DeferredAddressSpaceSink swaps later, both see it). var applier = new AddressSpaceApplier( addressSpaceSink, loggerFactory.CreateLogger(), provisioning: provisioning, historizedSubscriptions: historizedSubscriptions); var publishActor = system.ActorOf( OpcUaPublishActor.Props( sink: addressSpaceSink, serviceLevel: serviceLevel, localNode: roleInfo.LocalNode, dbFactory: dbFactory, applier: applier, dbHealthProbe: dbHealth), OpcUaPublishActorName); registry.Register(publishActor); // Per-node peer-probe supervisor — keeps one OPC UA TCP probe per OTHER non-Detached // driver node, so every node is continuously probed by all its peers. OpcUaPublishActor // consumes the resulting probe verdicts to learn this node's own reachability. var peerProbes = system.ActorOf( PeerProbeSupervisor.Props(roleInfo.LocalNode), PeerProbeSupervisorName); registry.Register(peerProbes); var driverHost = system.ActorOf( // ackRouter: under ClusterClient mode the node comm actor relays ApplyAcks across // the boundary; under Dps it stays null and the host publishes on the // deployment-acks topic exactly as before. DriverHostActor.Props(dbFactory, roleInfo.LocalNode, ackRouter: useClusterClient ? nodeComm : null, driverFactory: driverFactory, localRoles: roleInfo.LocalRoles, dependencyMux: mux, opcUaPublishActor: publishActor, healthPublisher: healthPublisher, virtualTagEvaluator: virtualTagEvaluator, historyWriter: historyWriter, loggerFactory: loggerFactory, scriptRootLogger: scriptRootLogger, invokerFactory: invokerFactory, deploymentArtifactCache: deploymentArtifactCache, redundancyRoleView: redundancyRoleView, replicationPeerHost: replicationPeerHost, fetchAndCacheMode: fetchAndCacheMode, artifactFetcher: artifactFetcher), DriverHostActorName); registry.Register(driverHost); var historian = system.ActorOf( HistorianAdapterActor.Props(historianSink, roleInfo.LocalNode), HistorianAdapterActorName); registry.Register(historian); }); return builder; } } /// Marker key types used by Akka.Hosting to resolve runtime actors from the registry. public sealed class DriverHostActorKey { } public sealed class DbHealthProbeActorKey { } public sealed class HistorianAdapterActorKey { } public sealed class DependencyMuxActorKey { } public sealed class OpcUaPublishActorKey { } public sealed class NodeCommunicationActorKey { } /// Marker key for the per-node ContinuousHistorizationRecorder (spawned only when /// ContinuousHistorization:Enabled=true and the gateway value-writer + outbox are registered). public sealed class ContinuousHistorizationRecorderKey { } /// Marker key for the per-node PeerProbeSupervisor. public sealed class PeerProbeSupervisorKey { }