Joseph Doherty
491 lines
20 KiB
C#
491 lines
20 KiB
C#
using System.Collections.Concurrent;
|
|
using Akka.Actor;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
|
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Communication;
|
|
|
|
/// <summary>
|
|
/// Default <see cref="ISiteAlarmLiveCache"/> — the DI singleton that owns the shared,
|
|
/// reference-counted, transient per-site live alarm cache (plan #10, Task 4). For each
|
|
/// site with active Alarm Summary viewers it runs ONE
|
|
/// <see cref="SiteAlarmAggregatorActor"/>, seeded by the existing debug-snapshot fan-out
|
|
/// (<see cref="CommunicationService.RequestDebugSnapshotAsync"/>) and kept warm on a
|
|
/// single site-wide, alarm-only gRPC stream. The actor holds the state and enforces
|
|
/// seed-then-stream ordering, dedup, placeholder coherence, failover re-seed and periodic
|
|
/// reconcile; this service is the DI façade: viewer reference-counting, a linger-delayed
|
|
/// last-out stop, the seed fan-out closure, and the published-snapshot store the page reads.
|
|
/// <para>
|
|
/// <b>No persistence</b> (locked <c>[PERM]</c>): nothing here writes to the database — the
|
|
/// cache lives only in the actor's memory on the active central node.
|
|
/// </para>
|
|
/// </summary>
|
|
public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
|
{
|
|
private static readonly IReadOnlyList<AlarmStateChanged> Empty = Array.Empty<AlarmStateChanged>();
|
|
|
|
private readonly IServiceProvider _serviceProvider;
|
|
private readonly CommunicationService _communicationService;
|
|
private readonly SiteStreamGrpcClientFactory _grpcFactory;
|
|
private readonly CommunicationOptions _options;
|
|
private readonly ILogger<SiteAlarmLiveCacheService> _logger;
|
|
|
|
private readonly object _lock = new();
|
|
private readonly Dictionary<int, SiteEntry> _sites = new();
|
|
private ActorSystem? _actorSystem;
|
|
|
|
/// <summary>
|
|
/// Initializes the live alarm cache service.
|
|
/// </summary>
|
|
/// <param name="serviceProvider">Root provider; scopes are created per snapshot fan-out to resolve repositories.</param>
|
|
/// <param name="communicationService">Central-side comms used to Ask each instance for a debug snapshot (the seed).</param>
|
|
/// <param name="grpcFactory">Factory for the per-site site-wide alarm gRPC client.</param>
|
|
/// <param name="options">Communication options (linger, reconcile interval, seed concurrency, subscriber cap).</param>
|
|
/// <param name="logger">Logger.</param>
|
|
public SiteAlarmLiveCacheService(
|
|
IServiceProvider serviceProvider,
|
|
CommunicationService communicationService,
|
|
SiteStreamGrpcClientFactory grpcFactory,
|
|
IOptions<CommunicationOptions> options,
|
|
ILogger<SiteAlarmLiveCacheService> logger)
|
|
{
|
|
_serviceProvider = serviceProvider;
|
|
_communicationService = communicationService;
|
|
_grpcFactory = grpcFactory;
|
|
_options = options.Value;
|
|
_logger = logger;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Supplies the <see cref="ActorSystem"/> once it exists (called from
|
|
/// <c>AkkaHostedService</c> on the active central node, mirroring
|
|
/// <see cref="DebugStreamService.SetActorSystem"/>). Until set, <see cref="Subscribe"/>
|
|
/// registers viewers but cannot start an aggregator, so the page keeps its poll fallback.
|
|
/// </summary>
|
|
/// <param name="actorSystem">The actor system to host per-site aggregators.</param>
|
|
public void SetActorSystem(ActorSystem actorSystem) => _actorSystem = actorSystem;
|
|
|
|
/// <inheritdoc />
|
|
public IDisposable Subscribe(int siteId, Action onChanged)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(onChanged);
|
|
|
|
var subscription = new Subscription(this, siteId, onChanged);
|
|
bool startNeeded;
|
|
SiteEntry entry;
|
|
|
|
lock (_lock)
|
|
{
|
|
if (!_sites.TryGetValue(siteId, out entry!))
|
|
{
|
|
entry = new SiteEntry(siteId);
|
|
_sites[siteId] = entry;
|
|
}
|
|
|
|
// A pending linger stop is now moot — a viewer arrived. Invalidate it.
|
|
entry.CancelLinger();
|
|
|
|
// Enforce the per-site viewer cap — fail SAFE: reject the excess viewer with a
|
|
// no-op disposable rather than throwing (this runs on a Blazor render path) or
|
|
// growing the subscriber list unbounded. The shared aggregator keeps serving the
|
|
// already-registered viewers; the rejected circuit just keeps its 15s poll
|
|
// fallback. Reaching the cap almost always means a Dispose leak or a runaway page.
|
|
if (entry.Subscribers.Count >= _options.LiveAlarmCacheMaxSubscribersPerSite)
|
|
{
|
|
_logger.LogWarning(
|
|
"Site {SiteId} live alarm cache is at its viewer cap ({Cap}); rejecting the new " +
|
|
"viewer (it will keep polling). This indicates a subscription leak or a very busy page.",
|
|
siteId, _options.LiveAlarmCacheMaxSubscribersPerSite);
|
|
return NoOpSubscription.Instance;
|
|
}
|
|
|
|
entry.Subscribers.Add(subscription);
|
|
|
|
// Start the aggregator on the first viewer (or if a prior start failed and left
|
|
// no actor). Guarded by Starting so concurrent first-viewers don't double-start.
|
|
startNeeded = entry.Actor is null && !entry.Starting;
|
|
if (startNeeded)
|
|
entry.Starting = true;
|
|
}
|
|
|
|
if (startNeeded)
|
|
_ = StartAggregatorAsync(entry);
|
|
|
|
return subscription;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public IReadOnlyList<AlarmStateChanged> GetCurrentAlarms(int siteId)
|
|
{
|
|
lock (_lock)
|
|
return _sites.TryGetValue(siteId, out var entry) ? entry.Current : Empty;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public bool IsLive(int siteId)
|
|
{
|
|
lock (_lock)
|
|
return _sites.TryGetValue(siteId, out var entry) && entry.HasPublished;
|
|
}
|
|
|
|
// ── Subscriber teardown ─────────────────────────────────────────────────────
|
|
|
|
private void Unsubscribe(Subscription subscription)
|
|
{
|
|
lock (_lock)
|
|
{
|
|
if (!_sites.TryGetValue(subscription.SiteId, out var entry))
|
|
return;
|
|
|
|
if (!entry.Subscribers.Remove(subscription))
|
|
return; // already removed (idempotent Dispose)
|
|
|
|
if (entry.Subscribers.Count == 0)
|
|
ScheduleLingerStop(entry);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Schedules a linger-delayed stop for a site whose last viewer just left. A
|
|
/// re-subscribe within the linger window cancels it (<see cref="SiteEntry.CancelLinger"/>).
|
|
/// Must be called under <see cref="_lock"/>.
|
|
/// </summary>
|
|
private void ScheduleLingerStop(SiteEntry entry)
|
|
{
|
|
var version = ++entry.LingerVersion;
|
|
entry.LingerTimer?.Dispose();
|
|
entry.LingerTimer = new Timer(
|
|
_ => LingerStop(entry.SiteId, version),
|
|
state: null,
|
|
dueTime: _options.LiveAlarmCacheLinger,
|
|
period: Timeout.InfiniteTimeSpan);
|
|
}
|
|
|
|
private void LingerStop(int siteId, int version)
|
|
{
|
|
IActorRef? actorToStop = null;
|
|
lock (_lock)
|
|
{
|
|
if (!_sites.TryGetValue(siteId, out var entry))
|
|
return;
|
|
// A viewer may have returned (version bumped) or re-subscribed (subscribers > 0)
|
|
// after this timer was armed — only stop if this is still the latest, empty state.
|
|
if (entry.LingerVersion != version || entry.Subscribers.Count != 0)
|
|
return;
|
|
|
|
actorToStop = entry.Actor;
|
|
entry.DisposeTimers();
|
|
_sites.Remove(siteId);
|
|
}
|
|
|
|
actorToStop?.Tell(new StopSiteAlarmAggregator());
|
|
_logger.LogDebug("Stopped live alarm aggregator for site {SiteId} (last viewer left)", siteId);
|
|
}
|
|
|
|
// ── Aggregator start ────────────────────────────────────────────────────────
|
|
|
|
private async Task StartAggregatorAsync(SiteEntry entry)
|
|
{
|
|
try
|
|
{
|
|
var system = _actorSystem;
|
|
if (system is null)
|
|
{
|
|
_logger.LogWarning(
|
|
"Live alarm cache asked to start site {SiteId} before the ActorSystem was set; " +
|
|
"the page will keep its poll fallback", entry.SiteId);
|
|
return;
|
|
}
|
|
|
|
var resolved = await ResolveSiteAsync(entry.SiteId);
|
|
if (resolved is null)
|
|
return; // logged inside ResolveSiteAsync
|
|
|
|
var (siteIdentifier, grpcA, grpcB) = resolved.Value;
|
|
var correlationId = $"site-alarm-{entry.SiteId}-{Guid.NewGuid():N}";
|
|
|
|
Func<CancellationToken, Task<IReadOnlyList<AlarmStateChanged>>> seedFn =
|
|
ct => FanOutSnapshotsAsync(entry.SiteId, ct);
|
|
Action<IReadOnlyList<AlarmStateChanged>> publish =
|
|
snapshot => OnPublish(entry, snapshot);
|
|
|
|
lock (_lock)
|
|
{
|
|
// Bail if the entry was torn down (last viewer left + linger fired) while we
|
|
// resolved the site, or if a racing start already created the actor.
|
|
if (!_sites.TryGetValue(entry.SiteId, out var current) || !ReferenceEquals(current, entry))
|
|
return;
|
|
if (entry.Actor is not null)
|
|
return;
|
|
|
|
var props = Props.Create(() => new SiteAlarmAggregatorActor(
|
|
siteIdentifier,
|
|
correlationId,
|
|
seedFn,
|
|
publish,
|
|
_grpcFactory,
|
|
grpcA,
|
|
grpcB,
|
|
_options.LiveAlarmCacheReconcileInterval,
|
|
_options.LiveAlarmCachePublishCoalesce));
|
|
|
|
entry.Actor = system.ActorOf(props, $"site-alarm-aggregator-{entry.SiteId}-{Guid.NewGuid():N}");
|
|
entry.StartRetryTimer?.Dispose();
|
|
entry.StartRetryTimer = null;
|
|
_logger.LogInformation("Started live alarm aggregator for site {SiteId} ({SiteIdentifier})",
|
|
entry.SiteId, siteIdentifier);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to start live alarm aggregator for site {SiteId}", entry.SiteId);
|
|
}
|
|
finally
|
|
{
|
|
lock (_lock)
|
|
{
|
|
entry.Starting = false;
|
|
|
|
// A transient start failure (ActorSystem not yet set, a DB blip resolving
|
|
// the site, ActorOf throwing) would otherwise leave this viewer cohort
|
|
// polling forever — nothing re-attempts until the NEXT Subscribe. Arm a
|
|
// bounded retry so the feature self-heals once the condition clears, as
|
|
// long as viewers remain and the entry is still live.
|
|
var stillWanted = _sites.TryGetValue(entry.SiteId, out var current)
|
|
&& ReferenceEquals(current, entry)
|
|
&& entry.Actor is null
|
|
&& entry.Subscribers.Count > 0;
|
|
if (stillWanted)
|
|
{
|
|
entry.Starting = true; // keep the guard closed so a Subscribe won't double-start
|
|
entry.StartRetryTimer?.Dispose();
|
|
entry.StartRetryTimer = new Timer(
|
|
_ => { _ = StartAggregatorAsync(entry); },
|
|
state: null,
|
|
dueTime: _options.LiveAlarmCacheReconcileInterval,
|
|
period: Timeout.InfiniteTimeSpan);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Resolves a site's identifier + both gRPC node endpoints. Returns <c>null</c> (and
|
|
/// logs) when the site is unknown or has no gRPC addresses configured, in which case
|
|
/// no aggregator is started and the page keeps polling.
|
|
/// </summary>
|
|
private async Task<(string SiteIdentifier, string GrpcA, string GrpcB)?> ResolveSiteAsync(int siteId)
|
|
{
|
|
using var scope = _serviceProvider.CreateScope();
|
|
var siteRepo = scope.ServiceProvider.GetRequiredService<ISiteRepository>();
|
|
var site = await siteRepo.GetSiteByIdAsync(siteId);
|
|
if (site is null)
|
|
{
|
|
_logger.LogWarning("Live alarm cache: site {SiteId} not found; not starting an aggregator", siteId);
|
|
return null;
|
|
}
|
|
|
|
if (string.IsNullOrWhiteSpace(site.GrpcNodeAAddress) || string.IsNullOrWhiteSpace(site.GrpcNodeBAddress))
|
|
{
|
|
_logger.LogWarning(
|
|
"Live alarm cache: site {SiteId} ({SiteIdentifier}) has no gRPC node addresses; " +
|
|
"not starting an aggregator", siteId, site.SiteIdentifier);
|
|
return null;
|
|
}
|
|
|
|
return (site.SiteIdentifier, site.GrpcNodeAAddress!, site.GrpcNodeBAddress!);
|
|
}
|
|
|
|
// ── Snapshot fan-out (the seed / reconcile) ─────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Fans out one debug-snapshot Ask per Enabled instance on the site (capped at
|
|
/// <see cref="CommunicationOptions.LiveAlarmCacheSeedConcurrency"/>) and returns every
|
|
/// snapshot's alarm rows — <b>including</b> <see cref="AlarmStateChanged.IsConfiguredPlaceholder"/>
|
|
/// placeholders. Best-effort: an instance that faults, times out, or reports
|
|
/// <see cref="DebugViewSnapshot.InstanceNotFound"/> contributes no rows rather than
|
|
/// failing the whole seed (mirrors <c>AlarmSummaryService</c>). Re-enumerates instances
|
|
/// each call so a reconcile picks up enable/disable/deploy/delete drift.
|
|
/// </summary>
|
|
private async Task<IReadOnlyList<AlarmStateChanged>> FanOutSnapshotsAsync(int siteId, CancellationToken ct)
|
|
{
|
|
string siteIdentifier;
|
|
List<string> enabledInstanceNames;
|
|
|
|
using (var scope = _serviceProvider.CreateScope())
|
|
{
|
|
var siteRepo = scope.ServiceProvider.GetRequiredService<ISiteRepository>();
|
|
var site = await siteRepo.GetSiteByIdAsync(siteId, ct);
|
|
if (site is null)
|
|
return Empty;
|
|
siteIdentifier = site.SiteIdentifier;
|
|
|
|
var instanceRepo = scope.ServiceProvider.GetRequiredService<ITemplateEngineRepository>();
|
|
var instances = await instanceRepo.GetInstancesBySiteIdAsync(siteId, ct);
|
|
enabledInstanceNames = instances
|
|
.Where(i => i.State == InstanceState.Enabled)
|
|
.Select(i => i.UniqueName)
|
|
.ToList();
|
|
}
|
|
|
|
if (enabledInstanceNames.Count == 0)
|
|
return Empty;
|
|
|
|
var rows = new ConcurrentBag<AlarmStateChanged>();
|
|
using var gate = new SemaphoreSlim(
|
|
Math.Max(1, _options.LiveAlarmCacheSeedConcurrency),
|
|
Math.Max(1, _options.LiveAlarmCacheSeedConcurrency));
|
|
|
|
var fetches = enabledInstanceNames.Select(name =>
|
|
FetchInstanceSnapshotAsync(siteIdentifier, name, gate, rows, ct));
|
|
await Task.WhenAll(fetches);
|
|
|
|
return rows.ToList();
|
|
}
|
|
|
|
private async Task FetchInstanceSnapshotAsync(
|
|
string siteIdentifier,
|
|
string instanceUniqueName,
|
|
SemaphoreSlim gate,
|
|
ConcurrentBag<AlarmStateChanged> rows,
|
|
CancellationToken ct)
|
|
{
|
|
await gate.WaitAsync(ct);
|
|
try
|
|
{
|
|
var request = new DebugSnapshotRequest(instanceUniqueName, Guid.NewGuid().ToString("N"));
|
|
var snapshot = await _communicationService.RequestDebugSnapshotAsync(siteIdentifier, request, ct);
|
|
if (snapshot.InstanceNotFound)
|
|
return;
|
|
|
|
foreach (var alarm in snapshot.AlarmStates)
|
|
rows.Add(alarm);
|
|
}
|
|
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
|
{
|
|
throw; // seed cancelled (actor stopping) — propagate
|
|
}
|
|
catch
|
|
{
|
|
// Degrade this one instance to "no rows" rather than failing the whole seed.
|
|
}
|
|
finally
|
|
{
|
|
gate.Release();
|
|
}
|
|
}
|
|
|
|
// ── Publish (actor thread → viewers) ────────────────────────────────────────
|
|
|
|
private void OnPublish(SiteEntry entry, IReadOnlyList<AlarmStateChanged> snapshot)
|
|
{
|
|
Subscription[] subscribers;
|
|
lock (_lock)
|
|
{
|
|
// Store the fresh immutable snapshot (readers get it lock-free-ish via GetCurrentAlarms).
|
|
entry.Current = snapshot;
|
|
entry.HasPublished = true;
|
|
subscribers = entry.Subscribers.ToArray();
|
|
}
|
|
|
|
// Invoke viewer callbacks OUTSIDE the lock — a Blazor InvokeAsync(StateHasChanged)
|
|
// must never re-enter our lock, and one faulting viewer must not stop the others.
|
|
foreach (var subscriber in subscribers)
|
|
{
|
|
try
|
|
{
|
|
subscriber.OnChanged();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex,
|
|
"A live alarm viewer callback threw for site {SiteId}; continuing", entry.SiteId);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Nested types ────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>Per-site bookkeeping. All mutable fields are guarded by the service's <c>_lock</c>.</summary>
|
|
private sealed class SiteEntry
|
|
{
|
|
public SiteEntry(int siteId) => SiteId = siteId;
|
|
|
|
public int SiteId { get; }
|
|
public List<Subscription> Subscribers { get; } = new();
|
|
public IActorRef? Actor { get; set; }
|
|
public bool Starting { get; set; }
|
|
|
|
/// <summary>Current published snapshot the page renders. Swapped as a whole immutable list.</summary>
|
|
public IReadOnlyList<AlarmStateChanged> Current { get; set; } = Empty;
|
|
|
|
/// <summary>True once the aggregator has seeded and published at least once.</summary>
|
|
public bool HasPublished { get; set; }
|
|
|
|
public Timer? LingerTimer { get; set; }
|
|
public int LingerVersion { get; set; }
|
|
|
|
/// <summary>Bounded retry timer armed when a start attempt failed to create the actor.</summary>
|
|
public Timer? StartRetryTimer { get; set; }
|
|
|
|
/// <summary>Cancels any pending linger stop (a viewer arrived).</summary>
|
|
public void CancelLinger()
|
|
{
|
|
LingerVersion++;
|
|
LingerTimer?.Dispose();
|
|
LingerTimer = null;
|
|
}
|
|
|
|
/// <summary>Disposes both timers — called when the entry is torn down.</summary>
|
|
public void DisposeTimers()
|
|
{
|
|
LingerTimer?.Dispose();
|
|
LingerTimer = null;
|
|
StartRetryTimer?.Dispose();
|
|
StartRetryTimer = null;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The handle returned when a Subscribe is rejected at the per-site viewer cap. It was
|
|
/// never registered, so <see cref="Dispose"/> is a genuine no-op — safe to dispose any
|
|
/// number of times and it never touches the site's subscriber list.
|
|
/// </summary>
|
|
private sealed class NoOpSubscription : IDisposable
|
|
{
|
|
public static readonly NoOpSubscription Instance = new();
|
|
private NoOpSubscription() { }
|
|
public void Dispose() { }
|
|
}
|
|
|
|
/// <summary>The disposable handed to a viewer; idempotent <see cref="Dispose"/> unregisters it.</summary>
|
|
private sealed class Subscription : IDisposable
|
|
{
|
|
private readonly SiteAlarmLiveCacheService _owner;
|
|
private int _disposed;
|
|
|
|
public Subscription(SiteAlarmLiveCacheService owner, int siteId, Action onChanged)
|
|
{
|
|
_owner = owner;
|
|
SiteId = siteId;
|
|
OnChanged = onChanged;
|
|
}
|
|
|
|
public int SiteId { get; }
|
|
public Action OnChanged { get; }
|
|
|
|
public void Dispose()
|
|
{
|
|
if (Interlocked.Exchange(ref _disposed, 1) == 0)
|
|
_owner.Unsubscribe(this);
|
|
}
|
|
}
|
|
}
|