feat(batch28): implement jetstream api dispatch and account request core
This commit is contained in:
Joseph Doherty
430
dotnet/src/ZB.MOM.NatsNet.Server/NatsServer.JetStreamApi.cs
Normal file
430
dotnet/src/ZB.MOM.NatsNet.Server/NatsServer.JetStreamApi.cs
Normal file
@@ -0,0 +1,430 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using ZB.MOM.NatsNet.Server.Internal;
|
||||
|
||||
namespace ZB.MOM.NatsNet.Server;
|
||||
|
||||
public sealed partial class NatsServer
|
||||
{
|
||||
private readonly Lock _jsApiSubsLock = new();
|
||||
private readonly List<JetStreamApiSubscription> _jsApiSubscriptions = [];
|
||||
private IpQueue<JsApiRoutedRequest>? _jsApiRoutedReqs;
|
||||
private IpQueue<DelayedApiResponse>? _delayedApiResponses;
|
||||
|
||||
internal void ProcessJSAPIRoutedRequests()
|
||||
{
|
||||
var queue = _jsApiRoutedReqs;
|
||||
if (queue is null)
|
||||
return;
|
||||
|
||||
var client = CreateInternalJetStreamClient();
|
||||
var token = _quitCts.Token;
|
||||
|
||||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!queue.Ch.WaitToReadAsync(token).AsTask().GetAwaiter().GetResult())
|
||||
break;
|
||||
|
||||
while (true)
|
||||
{
|
||||
var (request, ok) = queue.PopOne();
|
||||
if (!ok)
|
||||
break;
|
||||
|
||||
var start = DateTime.UtcNow;
|
||||
request.Subscription.Handler(client, request.Account, request.Subject, request.Reply, request.Message);
|
||||
var elapsed = DateTime.UtcNow - start;
|
||||
if (elapsed >= TimeSpan.FromSeconds(1))
|
||||
Warnf("Internal subscription on '{0}' took too long: {1}", request.Subject, elapsed);
|
||||
|
||||
var js = GetJetStreamState();
|
||||
if (js is not null)
|
||||
Interlocked.Add(ref js.ApiInflight, -1);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Warnf("JetStream routed API worker failed: {0}", ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal Exception? SetJetStreamExportSubs()
|
||||
{
|
||||
var js = GetJetStreamState();
|
||||
if (js is null)
|
||||
return new InvalidOperationException(JsApiErrors.NewJSNotEnabledError().Description ?? "jetstream not enabled");
|
||||
|
||||
var workers = Math.Min(Math.Max(Environment.ProcessorCount, 1), 16);
|
||||
|
||||
_jsApiRoutedReqs = IpQueue<JsApiRoutedRequest>.NewIPQueue("Routed JS API Requests");
|
||||
_delayedApiResponses = IpQueue<DelayedApiResponse>.NewIPQueue("Delayed JS API Responses");
|
||||
Interlocked.Exchange(ref js.QueueLimit, JsApiSubjects.JsDefaultRequestQueueLimit);
|
||||
|
||||
for (var i = 0; i < workers; i++)
|
||||
StartGoRoutine(ProcessJSAPIRoutedRequests);
|
||||
|
||||
StartGoRoutine(DelayedAPIResponder);
|
||||
|
||||
lock (_jsApiSubsLock)
|
||||
{
|
||||
_jsApiSubscriptions.Clear();
|
||||
_jsApiSubscriptions.Add(new JetStreamApiSubscription
|
||||
{
|
||||
Subject = JsApiSubjects.JsApiAccountInfo,
|
||||
Handler = JsAccountInfoRequest,
|
||||
});
|
||||
}
|
||||
|
||||
var sys = SystemAccount();
|
||||
if (sys is null)
|
||||
return null;
|
||||
|
||||
var err = sys.AddServiceExport(JsApiSubjects.JsAllApi, null);
|
||||
if (err is not null)
|
||||
Warnf("Error setting up jetstream service exports: {0}", err.Message);
|
||||
return err;
|
||||
}
|
||||
|
||||
internal JetStreamApiSubscription? MatchJetStreamApiSubscription(string subject)
|
||||
{
|
||||
lock (_jsApiSubsLock)
|
||||
{
|
||||
foreach (var sub in _jsApiSubscriptions)
|
||||
{
|
||||
if (JetStreamApi.SubjectMatches(sub.Subject, subject))
|
||||
return sub;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
internal (bool Queued, long Dropped) EnqueueJSApiRequest(JetStream js, JsApiRoutedRequest request)
|
||||
{
|
||||
var queue = _jsApiRoutedReqs;
|
||||
if (queue is null)
|
||||
return (false, 0);
|
||||
|
||||
var (pending, error) = queue.Push(request);
|
||||
var limit = Interlocked.Read(ref js.QueueLimit);
|
||||
if (limit <= 0)
|
||||
limit = JsApiSubjects.JsDefaultRequestQueueLimit;
|
||||
|
||||
if (error is null && pending < limit)
|
||||
return (true, 0);
|
||||
|
||||
RateLimitFormatWarnf("JetStream API queue limit reached, dropping {0} requests", pending);
|
||||
var drained = queue.Drain();
|
||||
if (drained > 0)
|
||||
Interlocked.Add(ref js.ApiInflight, -drained);
|
||||
|
||||
return (false, drained);
|
||||
}
|
||||
|
||||
internal void SendAPIResponse(ClientInfo? ci, Account acc, string subject, string reply, string request, string response)
|
||||
{
|
||||
acc.TrackAPI();
|
||||
if (!string.IsNullOrWhiteSpace(reply))
|
||||
_ = SendInternalAccountMsgWithReply(acc, reply, string.Empty, null, response, false);
|
||||
|
||||
SendJetStreamAPIAuditAdvisory(ci, acc, subject, request, response);
|
||||
}
|
||||
|
||||
internal void SendAPIErrResponse(ClientInfo? ci, Account acc, string subject, string reply, string request, string response)
|
||||
{
|
||||
acc.TrackAPIErr();
|
||||
if (!string.IsNullOrWhiteSpace(reply))
|
||||
_ = SendInternalAccountMsgWithReply(acc, reply, string.Empty, null, response, false);
|
||||
|
||||
SendJetStreamAPIAuditAdvisory(ci, acc, subject, request, response);
|
||||
}
|
||||
|
||||
internal void DelayedAPIResponder()
|
||||
{
|
||||
var queue = _delayedApiResponses;
|
||||
if (queue is null)
|
||||
return;
|
||||
|
||||
var token = _quitCts.Token;
|
||||
DelayedApiResponse? head = null;
|
||||
DelayedApiResponse? tail = null;
|
||||
|
||||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var (entry, ok) = queue.PopOne();
|
||||
if (!ok)
|
||||
break;
|
||||
JetStreamApi.AddDelayedResponse(ref head, ref tail, entry);
|
||||
}
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
if (head is not null && head.DeadlineUtc <= now)
|
||||
{
|
||||
var entry = head;
|
||||
head = head.Next;
|
||||
if (head is null)
|
||||
tail = null;
|
||||
|
||||
if (entry.RawResponse)
|
||||
_ = SendInternalAccountMsgWithReply(entry.Account, entry.Subject, string.Empty, entry.Header, entry.Response, false);
|
||||
else
|
||||
SendAPIErrResponse(entry.ClientInfo, entry.Account, entry.Subject, entry.Reply, entry.Request, entry.Response);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var wait = head is null
|
||||
? TimeSpan.FromSeconds(1)
|
||||
: head.DeadlineUtc - now;
|
||||
|
||||
if (wait < TimeSpan.Zero)
|
||||
wait = TimeSpan.Zero;
|
||||
|
||||
_ = Task.WhenAny(
|
||||
queue.Ch.WaitToReadAsync(token).AsTask(),
|
||||
Task.Delay(wait, token)).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Warnf("Delayed JetStream API responder failed: {0}", ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal void SendDelayedAPIErrResponse(
|
||||
ClientInfo? ci,
|
||||
Account acc,
|
||||
string subject,
|
||||
string reply,
|
||||
string request,
|
||||
string response,
|
||||
TimeSpan duration)
|
||||
{
|
||||
_delayedApiResponses?.Push(new DelayedApiResponse
|
||||
{
|
||||
ClientInfo = ci,
|
||||
Account = acc,
|
||||
Subject = subject,
|
||||
Reply = reply,
|
||||
Request = request,
|
||||
Response = response,
|
||||
DeadlineUtc = DateTime.UtcNow.Add(duration),
|
||||
RawResponse = false,
|
||||
});
|
||||
}
|
||||
|
||||
internal void SendDelayedErrResponse(Account acc, string subject, byte[]? header, string response, TimeSpan duration)
|
||||
{
|
||||
_delayedApiResponses?.Push(new DelayedApiResponse
|
||||
{
|
||||
Account = acc,
|
||||
Subject = subject,
|
||||
Reply = string.Empty,
|
||||
Header = header,
|
||||
Response = response,
|
||||
DeadlineUtc = DateTime.UtcNow.Add(duration),
|
||||
RawResponse = true,
|
||||
});
|
||||
}
|
||||
|
||||
internal (ClientInfo? ClientInfo, Account? Account, byte[] Header, byte[] Message, Exception? Error) GetRequestInfo(ClientConnection c, byte[] raw)
|
||||
{
|
||||
var (hdr, msg) = c.MsgParts(raw);
|
||||
|
||||
ClientInfo? clientInfo = null;
|
||||
var clientInfoBytes = NatsMessageHeaders.GetHeader(AccountEventConstants.ClientInfoHeader, hdr);
|
||||
if (clientInfoBytes is { Length: > 0 })
|
||||
{
|
||||
try
|
||||
{
|
||||
clientInfo = JsonSerializer.Deserialize<ClientInfo>(clientInfoBytes);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return (null, null, hdr, msg, ex);
|
||||
}
|
||||
}
|
||||
|
||||
Account? acc = null;
|
||||
var serviceAccount = clientInfo?.ServiceAccount();
|
||||
if (!string.IsNullOrWhiteSpace(serviceAccount))
|
||||
(acc, _) = LookupAccount(serviceAccount);
|
||||
|
||||
if (acc is null)
|
||||
{
|
||||
acc = c.Account() as Account;
|
||||
acc ??= SystemAccount();
|
||||
}
|
||||
|
||||
if (acc is null)
|
||||
return (clientInfo, null, hdr, msg, ServerErrors.ErrMissingAccount);
|
||||
|
||||
return (clientInfo, acc, hdr, msg, null);
|
||||
}
|
||||
|
||||
internal Exception? UnmarshalRequest<T>(ClientConnection c, Account acc, string subject, byte[] message, out T? value)
|
||||
{
|
||||
var strictMode = JetStreamConfig()?.Strict ?? true;
|
||||
value = JetStreamApi.DeserializeStrict<T>(message, strictMode);
|
||||
if (value is not null)
|
||||
return null;
|
||||
|
||||
var err = new InvalidOperationException("unable to deserialize JetStream API request");
|
||||
c.RateLimitWarnf("Invalid JetStream request '{0} > {1}': {2}", acc.Name, subject, err.Message);
|
||||
return err;
|
||||
}
|
||||
|
||||
internal Exception? UnmarshalRequest(ClientConnection c, Account acc, string subject, byte[] message, object destination)
|
||||
{
|
||||
var strictMode = JetStreamConfig()?.Strict ?? true;
|
||||
object? parsed;
|
||||
|
||||
try
|
||||
{
|
||||
parsed = JsonSerializer.Deserialize(message, destination.GetType(), new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow,
|
||||
});
|
||||
}
|
||||
catch (Exception ex) when (!strictMode)
|
||||
{
|
||||
try
|
||||
{
|
||||
parsed = JsonSerializer.Deserialize(message, destination.GetType());
|
||||
}
|
||||
catch
|
||||
{
|
||||
c.RateLimitWarnf("Invalid JetStream request '{0} > {1}': {2}", acc.Name, subject, ex.Message);
|
||||
return ex;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
c.RateLimitWarnf("Invalid JetStream request '{0} > {1}': {2}", acc.Name, subject, ex.Message);
|
||||
return ex;
|
||||
}
|
||||
|
||||
if (parsed is null)
|
||||
return new InvalidOperationException("empty JetStream API request body");
|
||||
|
||||
foreach (var property in destination.GetType().GetProperties())
|
||||
{
|
||||
if (!property.CanRead || !property.CanWrite)
|
||||
continue;
|
||||
property.SetValue(destination, property.GetValue(parsed));
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
internal void JsAccountInfoRequest(ClientConnection c, Account acc, string subject, string reply, byte[] rawMessage)
|
||||
{
|
||||
if (!JetStreamEnabled())
|
||||
return;
|
||||
|
||||
var (ci, requestAcc, hdr, msg, err) = GetRequestInfo(c, rawMessage);
|
||||
if (err is not null || requestAcc is null)
|
||||
{
|
||||
Warnf("Malformed JetStream API Request: {0}", Encoding.UTF8.GetString(msg));
|
||||
return;
|
||||
}
|
||||
|
||||
var response = new JsApiAccountInfoResponse
|
||||
{
|
||||
Type = JsApiSubjects.JsApiAccountInfoResponseType,
|
||||
};
|
||||
|
||||
var requiredApiLevel = NatsMessageHeaders.GetHeader(JsApiSubjects.JsRequiredApiLevel, hdr);
|
||||
var reqApiHeader = requiredApiLevel is null ? null : Encoding.ASCII.GetString(requiredApiLevel);
|
||||
if (JetStreamVersioning.ErrorOnRequiredApiLevel(reqApiHeader))
|
||||
{
|
||||
response.Error = JsApiErrors.NewJSRequiredApiLevelError();
|
||||
SendAPIErrResponse(ci, requestAcc, subject, reply, Encoding.UTF8.GetString(msg), JsonResponse(response));
|
||||
return;
|
||||
}
|
||||
|
||||
var (hasJetStream, shouldError) = requestAcc.CheckJetStream();
|
||||
if (!hasJetStream)
|
||||
{
|
||||
if (!shouldError)
|
||||
return;
|
||||
|
||||
response.Error = JsApiErrors.NewJSNotEnabledForAccountError();
|
||||
SendAPIErrResponse(ci, requestAcc, subject, reply, Encoding.UTF8.GetString(msg), JsonResponse(response));
|
||||
return;
|
||||
}
|
||||
|
||||
var stats = requestAcc.JetStreamUsage();
|
||||
response.Memory = stats.Memory;
|
||||
response.Store = stats.Store;
|
||||
response.ReservedMemory = stats.ReservedMemory;
|
||||
response.ReservedStore = stats.ReservedStore;
|
||||
response.Streams = stats.Streams;
|
||||
response.Consumers = stats.Consumers;
|
||||
response.Limits = stats.Limits;
|
||||
response.Domain = stats.Domain;
|
||||
response.Api = stats.Api;
|
||||
response.Tiers = stats.Tiers;
|
||||
|
||||
SendAPIResponse(ci, requestAcc, subject, reply, Encoding.UTF8.GetString(msg), JsonResponse(response));
|
||||
}
|
||||
|
||||
internal static string StreamNameFromSubject(string subject) => JetStreamApi.StreamNameFromSubject(subject);
|
||||
|
||||
internal static string ConsumerNameFromSubject(string subject) => JetStreamApi.ConsumerNameFromSubject(subject);
|
||||
|
||||
internal string JsonResponse(object response)
|
||||
{
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Serialize(response);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Warnf("Problem marshaling JSON for JetStream API: {0}", ex.Message);
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
internal void SendJetStreamAPIAuditAdvisory(ClientInfo? ci, Account acc, string subject, string request, string response)
|
||||
{
|
||||
_ = ci;
|
||||
_ = acc;
|
||||
_ = subject;
|
||||
_ = request;
|
||||
_ = response;
|
||||
}
|
||||
|
||||
private Exception? SendInternalAccountMsgWithReply(Account account, string subject, string reply, byte[]? header, string response, bool trackApi)
|
||||
{
|
||||
_ = trackApi;
|
||||
|
||||
try
|
||||
{
|
||||
var sendQueue = account.GetSendQueue() ?? NewSendQueue(account);
|
||||
SendQueue.Send(sendQueue, subject, reply, header ?? [], Encoding.UTF8.GetBytes(response));
|
||||
return null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user