using System.Collections.Concurrent; using Akka.Actor; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Communication.Actors; using ZB.MOM.WW.ScadaBridge.Communication.Grpc; namespace ZB.MOM.WW.ScadaBridge.Communication; /// /// Default — the DI singleton that owns the shared, /// reference-counted, transient per-site live alarm cache (plan #10, Task 4). For each /// site with active Alarm Summary viewers it runs ONE /// , seeded by the existing debug-snapshot fan-out /// () and kept warm on a /// single site-wide, alarm-only gRPC stream. The actor holds the state and enforces /// seed-then-stream ordering, dedup, placeholder coherence, failover re-seed and periodic /// reconcile; this service is the DI façade: viewer reference-counting, a linger-delayed /// last-out stop, the seed fan-out closure, and the published-snapshot store the page reads. /// /// No persistence (locked [PERM]): nothing here writes to the database — the /// cache lives only in the actor's memory on the active central node. /// /// public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache { private static readonly IReadOnlyList Empty = Array.Empty(); private readonly IServiceProvider _serviceProvider; private readonly CommunicationService _communicationService; private readonly SiteStreamGrpcClientFactory _grpcFactory; private readonly CommunicationOptions _options; private readonly ILogger _logger; private readonly object _lock = new(); private readonly Dictionary _sites = new(); private ActorSystem? _actorSystem; /// /// Initializes the live alarm cache service. /// /// Root provider; scopes are created per snapshot fan-out to resolve repositories. /// Central-side comms used to Ask each instance for a debug snapshot (the seed). /// Factory for the per-site site-wide alarm gRPC client. /// Communication options (linger, reconcile interval, seed concurrency, subscriber cap). /// Logger. public SiteAlarmLiveCacheService( IServiceProvider serviceProvider, CommunicationService communicationService, SiteStreamGrpcClientFactory grpcFactory, IOptions options, ILogger logger) { _serviceProvider = serviceProvider; _communicationService = communicationService; _grpcFactory = grpcFactory; _options = options.Value; _logger = logger; } /// /// Supplies the once it exists (called from /// AkkaHostedService on the active central node, mirroring /// ). Until set, /// registers viewers but cannot start an aggregator, so the page keeps its poll fallback. /// /// The actor system to host per-site aggregators. public void SetActorSystem(ActorSystem actorSystem) => _actorSystem = actorSystem; /// public IDisposable Subscribe(int siteId, Action onChanged) { ArgumentNullException.ThrowIfNull(onChanged); var subscription = new Subscription(this, siteId, onChanged); bool startNeeded; SiteEntry entry; lock (_lock) { if (!_sites.TryGetValue(siteId, out entry!)) { entry = new SiteEntry(siteId); _sites[siteId] = entry; } // A pending linger stop is now moot — a viewer arrived. Invalidate it. entry.CancelLinger(); // Enforce the per-site viewer cap — fail SAFE: reject the excess viewer with a // no-op disposable rather than throwing (this runs on a Blazor render path) or // growing the subscriber list unbounded. The shared aggregator keeps serving the // already-registered viewers; the rejected circuit just keeps its 15s poll // fallback. Reaching the cap almost always means a Dispose leak or a runaway page. if (entry.Subscribers.Count >= _options.LiveAlarmCacheMaxSubscribersPerSite) { _logger.LogWarning( "Site {SiteId} live alarm cache is at its viewer cap ({Cap}); rejecting the new " + "viewer (it will keep polling). This indicates a subscription leak or a very busy page.", siteId, _options.LiveAlarmCacheMaxSubscribersPerSite); return NoOpSubscription.Instance; } entry.Subscribers.Add(subscription); // Start the aggregator on the first viewer (or if a prior start failed and left // no actor). Guarded by Starting so concurrent first-viewers don't double-start. startNeeded = entry.Actor is null && !entry.Starting; if (startNeeded) entry.Starting = true; } if (startNeeded) _ = StartAggregatorAsync(entry); return subscription; } /// public IReadOnlyList GetCurrentAlarms(int siteId) { lock (_lock) return _sites.TryGetValue(siteId, out var entry) ? entry.Current : Empty; } /// public bool IsLive(int siteId) { lock (_lock) return _sites.TryGetValue(siteId, out var entry) && entry.HasPublished; } // ── Subscriber teardown ───────────────────────────────────────────────────── private void Unsubscribe(Subscription subscription) { lock (_lock) { if (!_sites.TryGetValue(subscription.SiteId, out var entry)) return; if (!entry.Subscribers.Remove(subscription)) return; // already removed (idempotent Dispose) if (entry.Subscribers.Count == 0) ScheduleLingerStop(entry); } } /// /// Schedules a linger-delayed stop for a site whose last viewer just left. A /// re-subscribe within the linger window cancels it (). /// Must be called under . /// private void ScheduleLingerStop(SiteEntry entry) { var version = ++entry.LingerVersion; entry.LingerTimer?.Dispose(); entry.LingerTimer = new Timer( _ => LingerStop(entry.SiteId, version), state: null, dueTime: _options.LiveAlarmCacheLinger, period: Timeout.InfiniteTimeSpan); } private void LingerStop(int siteId, int version) { IActorRef? actorToStop = null; lock (_lock) { if (!_sites.TryGetValue(siteId, out var entry)) return; // A viewer may have returned (version bumped) or re-subscribed (subscribers > 0) // after this timer was armed — only stop if this is still the latest, empty state. if (entry.LingerVersion != version || entry.Subscribers.Count != 0) return; actorToStop = entry.Actor; entry.DisposeTimers(); _sites.Remove(siteId); } actorToStop?.Tell(new StopSiteAlarmAggregator()); _logger.LogDebug("Stopped live alarm aggregator for site {SiteId} (last viewer left)", siteId); } // ── Aggregator start ──────────────────────────────────────────────────────── private async Task StartAggregatorAsync(SiteEntry entry) { try { var system = _actorSystem; if (system is null) { _logger.LogWarning( "Live alarm cache asked to start site {SiteId} before the ActorSystem was set; " + "the page will keep its poll fallback", entry.SiteId); return; } var resolved = await ResolveSiteAsync(entry.SiteId); if (resolved is null) return; // logged inside ResolveSiteAsync var (siteIdentifier, grpcA, grpcB) = resolved.Value; var correlationId = $"site-alarm-{entry.SiteId}-{Guid.NewGuid():N}"; Func>> seedFn = ct => FanOutSnapshotsAsync(entry.SiteId, ct); Action> publish = snapshot => OnPublish(entry, snapshot); lock (_lock) { // Bail if the entry was torn down (last viewer left + linger fired) while we // resolved the site, or if a racing start already created the actor. if (!_sites.TryGetValue(entry.SiteId, out var current) || !ReferenceEquals(current, entry)) return; if (entry.Actor is not null) return; var props = Props.Create(() => new SiteAlarmAggregatorActor( siteIdentifier, correlationId, seedFn, publish, _grpcFactory, grpcA, grpcB, _options.LiveAlarmCacheReconcileInterval, _options.LiveAlarmCachePublishCoalesce)); entry.Actor = system.ActorOf(props, $"site-alarm-aggregator-{entry.SiteId}-{Guid.NewGuid():N}"); entry.StartRetryTimer?.Dispose(); entry.StartRetryTimer = null; _logger.LogInformation("Started live alarm aggregator for site {SiteId} ({SiteIdentifier})", entry.SiteId, siteIdentifier); } } catch (Exception ex) { _logger.LogError(ex, "Failed to start live alarm aggregator for site {SiteId}", entry.SiteId); } finally { lock (_lock) { entry.Starting = false; // A transient start failure (ActorSystem not yet set, a DB blip resolving // the site, ActorOf throwing) would otherwise leave this viewer cohort // polling forever — nothing re-attempts until the NEXT Subscribe. Arm a // bounded retry so the feature self-heals once the condition clears, as // long as viewers remain and the entry is still live. var stillWanted = _sites.TryGetValue(entry.SiteId, out var current) && ReferenceEquals(current, entry) && entry.Actor is null && entry.Subscribers.Count > 0; if (stillWanted) { entry.Starting = true; // keep the guard closed so a Subscribe won't double-start entry.StartRetryTimer?.Dispose(); entry.StartRetryTimer = new Timer( _ => { _ = StartAggregatorAsync(entry); }, state: null, dueTime: _options.LiveAlarmCacheReconcileInterval, period: Timeout.InfiniteTimeSpan); } } } } /// /// Resolves a site's identifier + both gRPC node endpoints. Returns null (and /// logs) when the site is unknown or has no gRPC addresses configured, in which case /// no aggregator is started and the page keeps polling. /// private async Task<(string SiteIdentifier, string GrpcA, string GrpcB)?> ResolveSiteAsync(int siteId) { using var scope = _serviceProvider.CreateScope(); var siteRepo = scope.ServiceProvider.GetRequiredService(); var site = await siteRepo.GetSiteByIdAsync(siteId); if (site is null) { _logger.LogWarning("Live alarm cache: site {SiteId} not found; not starting an aggregator", siteId); return null; } if (string.IsNullOrWhiteSpace(site.GrpcNodeAAddress) || string.IsNullOrWhiteSpace(site.GrpcNodeBAddress)) { _logger.LogWarning( "Live alarm cache: site {SiteId} ({SiteIdentifier}) has no gRPC node addresses; " + "not starting an aggregator", siteId, site.SiteIdentifier); return null; } return (site.SiteIdentifier, site.GrpcNodeAAddress!, site.GrpcNodeBAddress!); } // ── Snapshot fan-out (the seed / reconcile) ───────────────────────────────── /// /// Fans out one debug-snapshot Ask per Enabled instance on the site (capped at /// ) and returns every /// snapshot's alarm rows — including /// placeholders. Best-effort: an instance that faults, times out, or reports /// contributes no rows rather than /// failing the whole seed (mirrors AlarmSummaryService). Re-enumerates instances /// each call so a reconcile picks up enable/disable/deploy/delete drift. /// private async Task> FanOutSnapshotsAsync(int siteId, CancellationToken ct) { string siteIdentifier; List enabledInstanceNames; using (var scope = _serviceProvider.CreateScope()) { var siteRepo = scope.ServiceProvider.GetRequiredService(); var site = await siteRepo.GetSiteByIdAsync(siteId, ct); if (site is null) return Empty; siteIdentifier = site.SiteIdentifier; var instanceRepo = scope.ServiceProvider.GetRequiredService(); var instances = await instanceRepo.GetInstancesBySiteIdAsync(siteId, ct); enabledInstanceNames = instances .Where(i => i.State == InstanceState.Enabled) .Select(i => i.UniqueName) .ToList(); } if (enabledInstanceNames.Count == 0) return Empty; var rows = new ConcurrentBag(); using var gate = new SemaphoreSlim( Math.Max(1, _options.LiveAlarmCacheSeedConcurrency), Math.Max(1, _options.LiveAlarmCacheSeedConcurrency)); var fetches = enabledInstanceNames.Select(name => FetchInstanceSnapshotAsync(siteIdentifier, name, gate, rows, ct)); await Task.WhenAll(fetches); return rows.ToList(); } private async Task FetchInstanceSnapshotAsync( string siteIdentifier, string instanceUniqueName, SemaphoreSlim gate, ConcurrentBag rows, CancellationToken ct) { await gate.WaitAsync(ct); try { var request = new DebugSnapshotRequest(instanceUniqueName, Guid.NewGuid().ToString("N")); var snapshot = await _communicationService.RequestDebugSnapshotAsync(siteIdentifier, request, ct); if (snapshot.InstanceNotFound) return; foreach (var alarm in snapshot.AlarmStates) rows.Add(alarm); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; // seed cancelled (actor stopping) — propagate } catch { // Degrade this one instance to "no rows" rather than failing the whole seed. } finally { gate.Release(); } } // ── Publish (actor thread → viewers) ──────────────────────────────────────── private void OnPublish(SiteEntry entry, IReadOnlyList snapshot) { Subscription[] subscribers; lock (_lock) { // Store the fresh immutable snapshot (readers get it lock-free-ish via GetCurrentAlarms). entry.Current = snapshot; entry.HasPublished = true; subscribers = entry.Subscribers.ToArray(); } // Invoke viewer callbacks OUTSIDE the lock — a Blazor InvokeAsync(StateHasChanged) // must never re-enter our lock, and one faulting viewer must not stop the others. foreach (var subscriber in subscribers) { try { subscriber.OnChanged(); } catch (Exception ex) { _logger.LogWarning(ex, "A live alarm viewer callback threw for site {SiteId}; continuing", entry.SiteId); } } } // ── Nested types ──────────────────────────────────────────────────────────── /// Per-site bookkeeping. All mutable fields are guarded by the service's _lock. private sealed class SiteEntry { public SiteEntry(int siteId) => SiteId = siteId; public int SiteId { get; } public List Subscribers { get; } = new(); public IActorRef? Actor { get; set; } public bool Starting { get; set; } /// Current published snapshot the page renders. Swapped as a whole immutable list. public IReadOnlyList Current { get; set; } = Empty; /// True once the aggregator has seeded and published at least once. public bool HasPublished { get; set; } public Timer? LingerTimer { get; set; } public int LingerVersion { get; set; } /// Bounded retry timer armed when a start attempt failed to create the actor. public Timer? StartRetryTimer { get; set; } /// Cancels any pending linger stop (a viewer arrived). public void CancelLinger() { LingerVersion++; LingerTimer?.Dispose(); LingerTimer = null; } /// Disposes both timers — called when the entry is torn down. public void DisposeTimers() { LingerTimer?.Dispose(); LingerTimer = null; StartRetryTimer?.Dispose(); StartRetryTimer = null; } } /// /// The handle returned when a Subscribe is rejected at the per-site viewer cap. It was /// never registered, so is a genuine no-op — safe to dispose any /// number of times and it never touches the site's subscriber list. /// private sealed class NoOpSubscription : IDisposable { public static readonly NoOpSubscription Instance = new(); private NoOpSubscription() { } public void Dispose() { } } /// The disposable handed to a viewer; idempotent unregisters it. private sealed class Subscription : IDisposable { private readonly SiteAlarmLiveCacheService _owner; private int _disposed; public Subscription(SiteAlarmLiveCacheService owner, int siteId, Action onChanged) { _owner = owner; SiteId = siteId; OnChanged = onChanged; } public int SiteId { get; } public Action OnChanged { get; } public void Dispose() { if (Interlocked.Exchange(ref _disposed, 1) == 0) _owner.Unsubscribe(this); } } }