feat(notification-outbox): async Notify.Send with status handle

Notify.To(list).Send(subject,body) now generates a NotificationId GUID,
enqueues a Notification-category message into the site Store-and-Forward
Engine, and returns the NotificationId immediately (Task<string>). The
NotificationId is the single idempotency key end-to-end: it is the S&F
message Id, it is carried inside the buffered NotificationSubmit payload,
and it is the id the forwarder submits to central.

NotificationForwarder now deserializes the buffered payload as a
NotificationSubmit and reads NotificationId from it (re-stamping only the
site-owned SourceSiteId / SourceInstanceId), instead of deriving the id
from StoreAndForwardMessage.Id.

Adds NotifyHelper.Status(id): queries central via the site communication
actor; reports the site-local Forwarding state while the notification is
still buffered at the site, maps central's response when found, and
Unknown otherwise. Adds a NotificationDeliveryStatus record.

SiteCommunicationActor gains a NotificationStatusQuery forwarding handler
mirroring NotificationSubmit. StoreAndForwardService.EnqueueAsync gains an
optional messageId parameter and exposes GetMessageByIdAsync.
This commit is contained in:
Joseph Doherty
2026-05-19 02:30:51 -04:00
parent 05614e037a
commit 3326bddeb0
9 changed files with 562 additions and 77 deletions

View File

@@ -190,6 +190,30 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
new ClusterClient.Send("/user/central-communication", msg), Sender);
});
// Notification Outbox: forward a Notify.Status query to the central cluster.
// The original Sender (the Notify helper's Ask) is forwarded as the
// ClusterClient.Send sender so the NotificationStatusResponse routes straight
// back to the waiting Ask, not here.
Receive<NotificationStatusQuery>(msg =>
{
if (_centralClient == null)
{
// No ClusterClient registered yet. Reply Found: false so Notify.Status
// falls back to the site S&F buffer to decide Forwarding vs Unknown.
_log.Warning(
"Cannot forward NotificationStatusQuery {0} — no central ClusterClient registered",
msg.NotificationId);
Sender.Tell(new NotificationStatusResponse(
msg.CorrelationId, Found: false, Status: "Unknown",
RetryCount: 0, LastError: null, DeliveredAt: null));
return;
}
_log.Debug("Forwarding NotificationStatusQuery {0} to central", msg.NotificationId);
_centralClient.Tell(
new ClusterClient.Send("/user/central-communication", msg), Sender);
});
// Internal: send heartbeat tick
Receive<SendHeartbeat>(_ => SendHeartbeatToCentral());

View File

@@ -8,6 +8,7 @@ using ScadaLink.Commons.Types;
using ScadaLink.HealthMonitoring;
using ScadaLink.SiteEventLogging;
using ScadaLink.SiteRuntime.Scripts;
using ScadaLink.StoreAndForward;
namespace ScadaLink.SiteRuntime.Actors;
@@ -78,6 +79,11 @@ public class ScriptExecutionActor : ReceiveActor
// starve the global pool and stall Akka dispatchers / HTTP handling.
var scheduler = ScriptExecutionScheduler.Shared(options);
// Notification Outbox: the site communication actor that Notify.Status queries
// central through. Resolved by actor path so the Notify helper does not need an
// IActorRef threaded all the way down from the host wiring.
var siteCommunicationActor = Context.System.ActorSelection("/user/site-communication");
// CTS must be created inside the async lambda so it outlives this method
_ = Task.Factory.StartNew(async () =>
{
@@ -91,14 +97,19 @@ public class ScriptExecutionActor : ReceiveActor
// Resolve integration services from DI (scoped lifetime)
IExternalSystemClient? externalSystemClient = null;
IDatabaseGateway? databaseGateway = null;
INotificationDeliveryService? notificationService = null;
// Notification Outbox: the S&F engine is a singleton; the site identity
// provider supplies the site id stamped on enqueued notifications.
StoreAndForwardService? storeAndForward = null;
var siteId = string.Empty;
if (serviceProvider != null)
{
serviceScope = serviceProvider.CreateScope();
externalSystemClient = serviceScope.ServiceProvider.GetService<IExternalSystemClient>();
databaseGateway = serviceScope.ServiceProvider.GetService<IDatabaseGateway>();
notificationService = serviceScope.ServiceProvider.GetService<INotificationDeliveryService>();
storeAndForward = serviceScope.ServiceProvider.GetService<StoreAndForwardService>();
siteId = serviceScope.ServiceProvider.GetService<ISiteIdentityProvider>()?.SiteId
?? string.Empty;
}
var context = new ScriptRuntimeContext(
@@ -112,7 +123,9 @@ public class ScriptExecutionActor : ReceiveActor
logger,
externalSystemClient,
databaseGateway,
notificationService);
storeAndForward,
siteCommunicationActor,
siteId);
var globals = new ScriptGlobals
{

View File

@@ -1,9 +1,13 @@
using System.Text.Json;
using Akka.Actor;
using Microsoft.Extensions.Logging;
using ScadaLink.Commons.Interfaces.Services;
using ScadaLink.Commons.Messages.Instance;
using ScadaLink.Commons.Messages.Notification;
using ScadaLink.Commons.Messages.ScriptExecution;
using ScadaLink.Commons.Types;
using ScadaLink.Commons.Types.Enums;
using ScadaLink.StoreAndForward;
namespace ScadaLink.SiteRuntime.Scripts;
@@ -46,9 +50,21 @@ public class ScriptRuntimeContext
private readonly IDatabaseGateway? _databaseGateway;
/// <summary>
/// WP-13: Notification delivery for Notify.To().Send().
/// Notification Outbox: the site Store-and-Forward Engine that <c>Notify.Send</c>
/// enqueues notifications into. The S&amp;F engine forwards them to central.
/// </summary>
private readonly INotificationDeliveryService? _notificationService;
private readonly StoreAndForwardService? _storeAndForward;
/// <summary>
/// Notification Outbox: the site communication actor that <c>Notify.Status</c>
/// queries central through (via the ClusterClient command/control transport).
/// </summary>
private readonly ICanTell? _siteCommunicationActor;
/// <summary>
/// Notification Outbox: this site's identifier, stamped on enqueued notifications.
/// </summary>
private readonly string _siteId;
public ScriptRuntimeContext(
IActorRef instanceActor,
@@ -61,7 +77,9 @@ public class ScriptRuntimeContext
ILogger logger,
IExternalSystemClient? externalSystemClient = null,
IDatabaseGateway? databaseGateway = null,
INotificationDeliveryService? notificationService = null)
StoreAndForwardService? storeAndForward = null,
ICanTell? siteCommunicationActor = null,
string siteId = "")
{
_instanceActor = instanceActor;
_self = self;
@@ -73,7 +91,9 @@ public class ScriptRuntimeContext
_logger = logger;
_externalSystemClient = externalSystemClient;
_databaseGateway = databaseGateway;
_notificationService = notificationService;
_storeAndForward = storeAndForward;
_siteCommunicationActor = siteCommunicationActor;
_siteId = siteId;
}
/// <summary>
@@ -183,10 +203,13 @@ public class ScriptRuntimeContext
public DatabaseHelper Database => new(_databaseGateway, _instanceName, _logger);
/// <summary>
/// WP-13: Provides access to notification delivery.
/// Notify.To("listName").Send("subject", "message")
/// Provides access to the Notification Outbox API.
/// <c>Notify.To("listName").Send("subject", "message")</c> enqueues a notification
/// for central delivery and returns its <c>NotificationId</c>;
/// <c>Notify.Status(id)</c> queries the delivery status of that notification.
/// </summary>
public NotifyHelper Notify => new(_notificationService, _instanceName, _logger);
public NotifyHelper Notify => new(
_storeAndForward, _siteCommunicationActor, _siteId, _instanceName, _askTimeout, _logger);
/// <summary>
/// Helper class for Scripts.CallShared() syntax.
@@ -319,54 +342,205 @@ public class ScriptRuntimeContext
}
/// <summary>
/// WP-13: Helper for Notify.To("listName").Send("subject", "message") syntax.
/// Notification Outbox: helper for the <c>Notify</c> script API.
///
/// In the outbox design the site no longer delivers notification email inline.
/// <c>Notify.To("listName").Send(...)</c> enqueues the notification into the site
/// Store-and-Forward Engine — which forwards it to central — and returns a
/// <c>NotificationId</c> handle immediately. <c>Notify.Status(id)</c> later queries
/// the delivery status of that notification.
/// </summary>
public class NotifyHelper
{
private readonly INotificationDeliveryService? _service;
private readonly StoreAndForwardService? _storeAndForward;
private readonly ICanTell? _siteCommunicationActor;
private readonly string _siteId;
private readonly string _instanceName;
private readonly TimeSpan _askTimeout;
private readonly ILogger _logger;
internal NotifyHelper(INotificationDeliveryService? service, string instanceName, ILogger logger)
internal NotifyHelper(
StoreAndForwardService? storeAndForward,
ICanTell? siteCommunicationActor,
string siteId,
string instanceName,
TimeSpan askTimeout,
ILogger logger)
{
_service = service;
_storeAndForward = storeAndForward;
_siteCommunicationActor = siteCommunicationActor;
_siteId = siteId;
_instanceName = instanceName;
_askTimeout = askTimeout;
_logger = logger;
}
/// <summary>
/// Selects the notification list to send to.
/// </summary>
public NotifyTarget To(string listName)
{
return new NotifyTarget(listName, _service, _instanceName, _logger);
return new NotifyTarget(
listName, _storeAndForward, _siteId, _instanceName, _logger);
}
/// <summary>
/// Queries the delivery status of a previously-sent notification.
///
/// The query is issued to central via the site communication actor. While the
/// notification is still buffered in the site Store-and-Forward Engine — central
/// has no row for it yet (<c>Found: false</c>) but the buffer still holds the id —
/// the status is reported as the site-local <c>Forwarding</c> state. If central
/// has a row, its status is mapped through verbatim. If central does not know the
/// id and it is not buffered locally, the status is <c>Unknown</c>.
/// </summary>
public async Task<NotificationDeliveryStatus> Status(string notificationId)
{
if (_siteCommunicationActor == null)
throw new InvalidOperationException(
"Notification status query is not available — site communication actor not wired");
var correlationId = Guid.NewGuid().ToString();
var query = new NotificationStatusQuery(correlationId, notificationId);
NotificationStatusResponse response;
try
{
response = await _siteCommunicationActor
.Ask<NotificationStatusResponse>(query, _askTimeout);
}
catch (Exception ex)
{
// Central could not be reached. Fall through to the buffer check: if the
// notification is still in the local S&F buffer it is Forwarding.
_logger.LogWarning(ex,
"Notification status query for {NotificationId} did not reach central",
notificationId);
response = new NotificationStatusResponse(
correlationId, Found: false, Status: "Unknown",
RetryCount: 0, LastError: null, DeliveredAt: null);
}
if (response.Found)
{
return new NotificationDeliveryStatus(
response.Status, response.RetryCount, response.LastError, response.DeliveredAt);
}
// Central has no row. If the notification is still buffered at the site it
// is in transit — report the site-local Forwarding state. Otherwise it is
// genuinely unknown (never sent, or already forwarded and central lost it).
if (_storeAndForward != null)
{
var buffered = await _storeAndForward.GetMessageByIdAsync(notificationId);
if (buffered != null)
{
return new NotificationDeliveryStatus(
"Forwarding", buffered.RetryCount, buffered.LastError, DeliveredAt: null);
}
}
return new NotificationDeliveryStatus("Unknown", 0, null, null);
}
}
/// <summary>
/// WP-13: Target for Notify.To("listName").Send("subject", "message").
/// Notification Outbox: target for <c>Notify.To("listName").Send(...)</c>.
/// </summary>
public class NotifyTarget
{
private readonly string _listName;
private readonly INotificationDeliveryService? _service;
private readonly StoreAndForwardService? _storeAndForward;
private readonly string _siteId;
private readonly string _instanceName;
private readonly ILogger _logger;
internal NotifyTarget(string listName, INotificationDeliveryService? service, string instanceName, ILogger logger)
internal NotifyTarget(
string listName,
StoreAndForwardService? storeAndForward,
string siteId,
string instanceName,
ILogger logger)
{
_listName = listName;
_service = service;
_storeAndForward = storeAndForward;
_siteId = siteId;
_instanceName = instanceName;
_logger = logger;
}
public async Task<NotificationResult> Send(
/// <summary>
/// Enqueues a notification for central delivery and returns its
/// <c>NotificationId</c> immediately.
///
/// The notification is buffered into the site Store-and-Forward Engine under the
/// <see cref="StoreAndForwardCategory.Notification"/> category; the S&amp;F
/// engine's <c>NotificationForwarder</c> forwards it to central and treats
/// central's ack as the delivery outcome. The returned <c>NotificationId</c> is
/// the single idempotency key end-to-end: it is the S&amp;F message id, it is
/// carried inside the buffered payload, and it is the id the forwarder submits to
/// central. Pass it to <see cref="NotifyHelper.Status"/> to track delivery.
/// </summary>
public async Task<string> Send(
string subject,
string message,
CancellationToken cancellationToken = default)
{
if (_service == null)
throw new InvalidOperationException("Notification service not available");
if (_storeAndForward == null)
throw new InvalidOperationException(
"Notification store-and-forward engine not available");
return await _service.SendAsync(_listName, subject, message, _instanceName, cancellationToken);
// The script controls the idempotency key: generate the NotificationId here,
// use it as the S&F message id, and carry it inside the buffered payload so
// the forwarder submits the same id to central on every retry.
var notificationId = Guid.NewGuid().ToString("N");
var payload = new NotificationSubmit(
NotificationId: notificationId,
ListName: _listName,
Subject: subject,
Body: message,
// SourceSiteId is re-stamped by the forwarder from its own site id; this
// value is the best-effort site id known to the script runtime.
SourceSiteId: _siteId,
SourceInstanceId: _instanceName,
// SourceScript: the script runtime does not currently thread the script
// name down to the Notify helper; left null until that wiring exists.
SourceScript: null,
SiteEnqueuedAt: DateTimeOffset.UtcNow);
var payloadJson = JsonSerializer.Serialize(payload);
// The S&F engine assigns its own GUID to the message; pin the message id to
// the NotificationId so the buffer can be queried by it (Notify.Status) and
// the forwarder's idempotency key matches the buffered row.
await _storeAndForward.EnqueueAsync(
StoreAndForwardCategory.Notification,
target: _listName,
payloadJson: payloadJson,
originInstanceName: _instanceName,
messageId: notificationId);
_logger.LogDebug(
"Notify enqueued notification {NotificationId} to list '{List}' for central delivery",
notificationId, _listName);
return notificationId;
}
}
}
/// <summary>
/// Notification Outbox: the delivery status of a notification, as returned to a
/// script by <c>Notify.Status(id)</c>.
///
/// <see cref="Status"/> is either a central status (<c>Pending</c>, <c>Retrying</c>,
/// <c>Delivered</c>, <c>Parked</c>, <c>Discarded</c>), the site-local <c>Forwarding</c>
/// state (the notification is still buffered at the site and has not yet been
/// forwarded/acked), or <c>Unknown</c> (no central row and not buffered locally).
/// </summary>
public record NotificationDeliveryStatus(
string Status,
int RetryCount,
string? LastError,
DateTimeOffset? DeliveredAt);

View File

@@ -87,21 +87,28 @@ public sealed class NotificationForwarder
}
/// <summary>
/// Maps a buffered S&amp;F notification message onto a <see cref="NotificationSubmit"/>,
/// returning <c>false</c> if the payload is unreadable.
/// The <see cref="NotificationSubmit.NotificationId"/> is the central idempotency
/// key and must be stable across every retry of the same buffered message, so it is
/// derived from <see cref="StoreAndForwardMessage.Id"/> — a stable GUID assigned
/// once at enqueue time.
/// Maps a buffered S&amp;F notification message onto the <see cref="NotificationSubmit"/>
/// forwarded to central, returning <c>false</c> if the payload is unreadable.
///
/// The buffered payload IS a serialized <see cref="NotificationSubmit"/> written by
/// the site <c>Notify.Send</c> enqueue path (Task 19). Its
/// <see cref="NotificationSubmit.NotificationId"/> is the central idempotency key —
/// it was generated by the script, equals the buffered row's
/// <see cref="StoreAndForwardMessage.Id"/>, and is stable across every retry. The
/// forwarder forwards the payload as-is except that it re-stamps the fields it
/// authoritatively owns: <see cref="NotificationSubmit.SourceSiteId"/> (this site's
/// id) and <see cref="NotificationSubmit.SourceInstanceId"/> (the buffered row's
/// origin instance), and it falls the list name back to the S&amp;F
/// <see cref="StoreAndForwardMessage.Target"/> when the payload list name is blank.
/// </summary>
private bool TryBuildSubmit(StoreAndForwardMessage message, out NotificationSubmit submit)
{
submit = null!;
BufferedNotificationPayload? payload;
NotificationSubmit? payload;
try
{
payload = JsonSerializer.Deserialize<BufferedNotificationPayload>(message.PayloadJson);
payload = JsonSerializer.Deserialize<NotificationSubmit>(message.PayloadJson);
}
catch (JsonException)
{
@@ -113,30 +120,25 @@ public sealed class NotificationForwarder
return false;
}
submit = new NotificationSubmit(
NotificationId: message.Id,
// A null OR empty/blank ListName falls back to the S&F Target — matching the
// empty-string guard the former SMTP handler (NotificationDeliveryService)
// applied, so an empty list name is never forwarded to central.
ListName: string.IsNullOrEmpty(payload.ListName) ? message.Target : payload.ListName,
Subject: payload.Subject ?? string.Empty,
Body: payload.Message ?? string.Empty,
SourceSiteId: _sourceSiteId,
SourceInstanceId: message.OriginInstanceName,
// The buffered payload does not currently carry the originating script;
// Task 19 (the enqueue side) will add it. Null until then.
SourceScript: null,
SiteEnqueuedAt: message.CreatedAt);
submit = payload with
{
// The NotificationId is the script-generated idempotency key carried in the
// payload. Defend against a payload missing it by falling back to the
// buffered row id, which the enqueue path pins to the same value.
NotificationId = string.IsNullOrEmpty(payload.NotificationId)
? message.Id
: payload.NotificationId,
// A null OR empty/blank ListName falls back to the S&F Target — so an empty
// list name is never forwarded to central.
ListName = string.IsNullOrEmpty(payload.ListName) ? message.Target : payload.ListName,
// SourceSiteId/SourceInstanceId are authoritatively owned by the site: the
// forwarder knows the real site id, and the buffered row records the origin
// instance even after the instance is deleted.
SourceSiteId = _sourceSiteId,
SourceInstanceId = message.OriginInstanceName,
};
return true;
}
/// <summary>
/// Mirrors the payload shape written by the site notification enqueue path
/// (<c>{ ListName, Subject, Message }</c>). Kept private to this forwarder — Task 19
/// will reshape the enqueue payload, at which point this is updated alongside it.
/// </summary>
private sealed record BufferedNotificationPayload(
string? ListName, string? Subject, string? Message);
}
/// <summary>

View File

@@ -149,6 +149,13 @@ public class StoreAndForwardService
/// When <c>false</c>, the caller has already made its own delivery attempt and the
/// message is buffered directly for the retry sweep (the handler is not invoked here).
/// </param>
/// <param name="messageId">
/// An explicit, caller-supplied message id. <c>null</c> (the default) makes the
/// service mint a fresh GUID. The Notification Outbox enqueue path supplies its own
/// id so the script-generated <c>NotificationId</c> is the single idempotency key —
/// it is the buffered row's <see cref="StoreAndForwardMessage.Id"/>, it is carried
/// inside the payload, and it is the id the forwarder submits to central.
/// </param>
public async Task<StoreAndForwardResult> EnqueueAsync(
StoreAndForwardCategory category,
string target,
@@ -156,11 +163,12 @@ public class StoreAndForwardService
string? originInstanceName = null,
int? maxRetries = null,
TimeSpan? retryInterval = null,
bool attemptImmediateDelivery = true)
bool attemptImmediateDelivery = true,
string? messageId = null)
{
var message = new StoreAndForwardMessage
{
Id = Guid.NewGuid().ToString("N"),
Id = messageId ?? Guid.NewGuid().ToString("N"),
Category = category,
Target = target,
PayloadJson = payloadJson,
@@ -430,6 +438,17 @@ public class StoreAndForwardService
return await _storage.GetMessageCountByOriginInstanceAsync(instanceName);
}
/// <summary>
/// Notification Outbox: looks up a buffered message by its id, or <c>null</c> if it
/// is not (or no longer) in the buffer. <c>Notify.Status</c> uses this to detect a
/// notification still in transit at the site — central reports it not-found while
/// the S&amp;F buffer still holds it, which is the site-local <c>Forwarding</c> state.
/// </summary>
public async Task<StoreAndForwardMessage?> GetMessageByIdAsync(string messageId)
{
return await _storage.GetMessageByIdAsync(messageId);
}
/// <summary>
/// WP-14: Raises the S&amp;F activity notification. StoreAndForward-009: the
/// delegate is snapshotted (so a concurrent unsubscribe cannot NRE) and every