using Akka.Actor; using Microsoft.Extensions.Logging; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors; /// /// Runs on EVERY site node (NOT a singleton) so a standby that was DOWN during a deploy /// self-heals on its next restart. On startup the actor performs one best-effort /// reconciliation pass: /// /// read the node's local deployed inventory from SQLite, /// report it to central via the SiteCommunicationActor Ask /// (), /// fetch each gap item's config over HTTP and guarded-write it, and /// LOG (never delete) any orphan the node still has but central no longer deploys. /// /// /// /// /// Best-effort throughout. A central-unreachable / timed-out Ask is caught, logged /// at Warning, and the pass simply ends — reconcile re-runs on the next node startup; it is a /// self-heal, not a critical path. A per-item fetch/write failure is caught and logged, then /// the remaining gap items continue (one bad item must not abort the rest). The actor never /// crashes on these failures. /// /// /// The pass runs after a small startup delay (so the central ClusterClient has time to /// register) and is driven entirely off the actor thread: the Ask + fetch + write happen in /// an awaited continuation whose summary is captured in an internal message /// piped back to Self. The actor thread never blocks. /// /// /// The site does NOT carry the central fetch base URL in its own config — it uses /// from central's reply. /// /// public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers { private const string StartupTimerKey = "reconcile-startup"; private readonly SiteStorageService _storage; private readonly IDeploymentConfigFetcher _configFetcher; private readonly IActorRef _siteCommunicationActor; private readonly string _siteIdentifier; private readonly string _nodeId; private readonly ILogger _logger; private readonly TimeSpan _initialDelay; private readonly TimeSpan _askTimeout; /// Akka timer scheduler injected by the framework via . public ITimerScheduler Timers { get; set; } = null!; /// /// Initializes the per-node startup-reconciliation actor. /// /// Site-local SQLite store — read for the inventory, written for the gap. /// Fetches a deployment's flattened config JSON from central over HTTP. /// /// The site's SiteCommunicationActor; it forwards the /// over the registered central ClusterClient and routes /// the back to this actor's Ask. /// /// This node's site identifier (resolved by central). /// This node's semantic id (e.g. node-a/node-b), for logging/diagnostics. /// Logger. /// /// Delay before the single startup pass, giving the central ClusterClient time to register. /// Defaults to 5 seconds. /// /// Round-trip timeout for the reconcile Ask to central. Defaults to 30 seconds. public SiteReconciliationActor( SiteStorageService storage, IDeploymentConfigFetcher configFetcher, IActorRef siteCommunicationActor, string siteIdentifier, string nodeId, ILogger logger, TimeSpan? initialDelay = null, TimeSpan? askTimeout = null) { _storage = storage; _configFetcher = configFetcher; _siteCommunicationActor = siteCommunicationActor; _siteIdentifier = siteIdentifier; _nodeId = nodeId; _logger = logger; _initialDelay = initialDelay ?? TimeSpan.FromSeconds(5); _askTimeout = askTimeout ?? TimeSpan.FromSeconds(30); Receive(_ => RunReconcilePassAsync().PipeTo(Self)); Receive(HandleReconcilePassResult); // Defensive: RunReconcilePassAsync is designed never to throw (it returns a faulted // ReconcilePassResult instead), but if anything unexpected faults the piped Task the // Status.Failure would otherwise go to dead letters silently. Log it instead. Receive(f => _logger.LogWarning(f.Cause, "Reconcile pass faulted unexpectedly for site {Site} node {Node}", _siteIdentifier, _nodeId)); } /// protected override void PreStart() { base.PreStart(); // One-shot pass after a small delay so the central ClusterClient can register first. // Non-blocking: the timer fires RunReconcile back onto this actor's mailbox. Timers.StartSingleTimer(StartupTimerKey, RunReconcile.Instance, _initialDelay); _logger.LogInformation( "SiteReconciliationActor started for site {Site} node {Node}; startup reconcile scheduled in {Delay}", _siteIdentifier, _nodeId, _initialDelay); } /// /// Runs the full reconcile pass off the actor thread. Never throws: a central-unreachable /// Ask (or any other top-level failure) is captured as a faulted /// ; per-item fetch/write failures are caught per item so /// the rest of the gap still applies. /// private async Task RunReconcilePassAsync() { Dictionary localMap; try { var configs = await _storage.GetAllDeployedConfigsAsync().ConfigureAwait(false); localMap = new Dictionary(configs.Count, StringComparer.Ordinal); foreach (var c in configs) localMap[c.InstanceUniqueName] = c.RevisionHash; } catch (Exception ex) { return ReconcilePassResult.Faulted(ex); } // Report inventory to central and get fresh fetch tokens for the gap. Best-effort: // a central-unreachable / timed-out Ask faults here and is reported as a faulted pass // (logged Warning; reconcile re-runs next startup). ReconcileSiteResponse response; try { response = await _siteCommunicationActor .Ask( new ReconcileSiteRequest(_siteIdentifier, _nodeId, localMap), _askTimeout) .ConfigureAwait(false); } catch (Exception ex) { return ReconcilePassResult.Faulted(ex); } var fetched = 0; var failed = 0; // Fetch + guarded-write each gap item. Per-item failure must not abort the rest. foreach (var item in response.Gap) { try { var configJson = await _configFetcher .FetchAsync(response.CentralFetchBaseUrl, item.DeploymentId, item.FetchToken, CancellationToken.None) .ConfigureAwait(false); await _storage.StoreDeployedConfigIfNewerAsync( item.InstanceUniqueName, configJson, item.DeploymentId, item.RevisionHash, item.IsEnabled) .ConfigureAwait(false); fetched++; _logger.LogInformation( "Reconcile: fetched + stored config for {Instance} (deployment {DeploymentId}, rev {Revision})", item.InstanceUniqueName, item.DeploymentId, item.RevisionHash); } catch (DeploymentConfigFetchException ex) when (ex.IsSuperseded) { // 404 = superseded/expired between staging and fetch; a newer deploy will // replicate it. Not a failure — skip quietly. _logger.LogInformation( "Reconcile: skip {Instance} (deployment {DeploymentId}) — superseded/expired", item.InstanceUniqueName, item.DeploymentId); } catch (Exception ex) { failed++; _logger.LogError(ex, "Reconcile: failed to fetch/store config for {Instance} (deployment {DeploymentId}) — continuing with remaining items", item.InstanceUniqueName, item.DeploymentId); } } // Orphans: present locally but no longer deployed at central. LOG only — never delete // (a stale local row is harmless; deleting risks dropping a config a later deploy needs). foreach (var name in response.OrphanNames) { _logger.LogWarning( "Reconcile: local instance {Instance} is no longer deployed at central — leaving in place; manual cleanup may be needed", name); } return ReconcilePassResult.Completed(fetched, failed, response.OrphanNames.Count); } private void HandleReconcilePassResult(ReconcilePassResult result) { if (result.Error != null) { // Best-effort: a failed pass (central unreachable, Ask timeout, local read error) is // logged at Warning and the actor stays alive. Reconcile re-runs on the next startup. _logger.LogWarning(result.Error, "Reconcile pass for site {Site} node {Node} did not complete (central unreachable or read error) — will retry on next startup", _siteIdentifier, _nodeId); return; } _logger.LogInformation( "Reconcile pass for site {Site} node {Node} complete: {Fetched} fetched, {Failed} failed, {Orphans} orphan(s)", _siteIdentifier, _nodeId, result.Fetched, result.Failed, result.Orphans); } // ── Internal messages ── /// Self-tick that drives the one-shot startup reconcile pass. private sealed class RunReconcile { public static readonly RunReconcile Instance = new(); private RunReconcile() { } } /// Summary of one reconcile pass, piped to Self for logging. private sealed record ReconcilePassResult(int Fetched, int Failed, int Orphans, Exception? Error) { /// Creates a result for a reconcile pass that ran to completion. /// Number of instances successfully fetched/reconciled. /// Number of instances that failed to reconcile. /// Number of local instances no longer deployed at central. /// A completed with no error. public static ReconcilePassResult Completed(int fetched, int failed, int orphans) => new(fetched, failed, orphans, null); /// Creates a result for a reconcile pass that failed before completing. /// The exception that caused the pass to fail. /// A faulted carrying the error. public static ReconcilePassResult Faulted(Exception error) => new(0, 0, 0, error); } }