feat(mesh-phase5): node-side TelemetryStreamService (hub -> server-streaming)
Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,167 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Node-side projection of a domain <see cref="TelemetryItem"/> onto its wire
|
||||
/// <see cref="TelemetryEvent"/> envelope (per-cluster mesh Phase 5). The exact reverse of
|
||||
/// <c>telemetry.proto</c>: it transcribes every domain field onto the matching proto field and
|
||||
/// selects the envelope's <c>oneof</c> arm from the <see cref="TelemetryItem"/> subtype.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>DateTime → Timestamp is Kind-defensive.</b> <see cref="Timestamp.FromDateTime"/> throws
|
||||
/// unless the source <see cref="DateTime.Kind"/> is <see cref="DateTimeKind.Utc"/>. Producers
|
||||
/// use <see cref="DateTime.UtcNow"/> today, but a stray <see cref="DateTimeKind.Local"/> or
|
||||
/// <see cref="DateTimeKind.Unspecified"/> value must never crash the stream — so a Local value
|
||||
/// is converted and an Unspecified value is assumed already-UTC (matching the producer intent
|
||||
/// and preserving the instant on any machine).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Nullable presence is preserved.</b> A null nullable-<c>DateTime</c> leaves the proto
|
||||
/// Timestamp unset (absent == null per the contract); a null <c>optional string</c> leaves the
|
||||
/// proto field unset so the generated <c>Has…</c> presence distinguishes null from "".
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class TelemetryProtoMapNode
|
||||
{
|
||||
/// <summary>Projects <paramref name="item"/> onto a wire <see cref="TelemetryEvent"/>.</summary>
|
||||
/// <param name="item">The domain telemetry item to project.</param>
|
||||
/// <param name="correlationId">Stream correlation id echoed on every envelope.</param>
|
||||
/// <returns>The wire envelope with the matching <c>oneof</c> arm populated.</returns>
|
||||
public static TelemetryEvent ToProto(TelemetryItem item, string correlationId)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(item);
|
||||
|
||||
return item switch
|
||||
{
|
||||
TelemetryItem.Alarm a => new TelemetryEvent
|
||||
{
|
||||
CorrelationId = correlationId,
|
||||
AlarmTransition = MapAlarm(a.E),
|
||||
},
|
||||
TelemetryItem.Script s => new TelemetryEvent
|
||||
{
|
||||
CorrelationId = correlationId,
|
||||
ScriptLog = MapScript(s.E),
|
||||
},
|
||||
TelemetryItem.Health h => new TelemetryEvent
|
||||
{
|
||||
CorrelationId = correlationId,
|
||||
DriverHealth = MapHealth(h.E),
|
||||
},
|
||||
TelemetryItem.Resilience r => new TelemetryEvent
|
||||
{
|
||||
CorrelationId = correlationId,
|
||||
DriverResilience = MapResilience(r.E),
|
||||
},
|
||||
_ => throw new ArgumentOutOfRangeException(
|
||||
nameof(item), item.GetType().Name, "Unknown TelemetryItem subtype"),
|
||||
};
|
||||
}
|
||||
|
||||
private static AlarmTransition MapAlarm(AlarmTransitionEvent e)
|
||||
{
|
||||
var msg = new AlarmTransition
|
||||
{
|
||||
AlarmId = e.AlarmId,
|
||||
EquipmentPath = e.EquipmentPath,
|
||||
AlarmName = e.AlarmName,
|
||||
TransitionKind = e.TransitionKind,
|
||||
Severity = e.Severity,
|
||||
Message = e.Message,
|
||||
User = e.User,
|
||||
TimestampUtc = ToUtcTimestamp(e.TimestampUtc),
|
||||
AlarmTypeName = e.AlarmTypeName,
|
||||
};
|
||||
|
||||
if (e.Comment is not null)
|
||||
msg.Comment = e.Comment;
|
||||
if (e.HistorizeToAveva is not null)
|
||||
msg.HistorizeToAveva = e.HistorizeToAveva.Value;
|
||||
msg.ReferencingEquipmentPaths.AddRange(e.ReferencingEquipmentPaths ?? Enumerable.Empty<string>());
|
||||
|
||||
return msg;
|
||||
}
|
||||
|
||||
private static ScriptLog MapScript(ScriptLogEntry e)
|
||||
{
|
||||
var msg = new ScriptLog
|
||||
{
|
||||
ScriptId = e.ScriptId,
|
||||
Level = e.Level,
|
||||
Message = e.Message,
|
||||
TimestampUtc = ToUtcTimestamp(e.TimestampUtc),
|
||||
};
|
||||
|
||||
if (e.VirtualTagId is not null)
|
||||
msg.VirtualTagId = e.VirtualTagId;
|
||||
if (e.AlarmId is not null)
|
||||
msg.AlarmId = e.AlarmId;
|
||||
if (e.EquipmentId is not null)
|
||||
msg.EquipmentId = e.EquipmentId;
|
||||
|
||||
return msg;
|
||||
}
|
||||
|
||||
private static DriverHealth MapHealth(DriverHealthChanged e)
|
||||
{
|
||||
var msg = new DriverHealth
|
||||
{
|
||||
ClusterId = e.ClusterId,
|
||||
DriverInstanceId = e.DriverInstanceId,
|
||||
State = e.State,
|
||||
ErrorCount5Min = e.ErrorCount5Min,
|
||||
PublishedUtc = ToUtcTimestamp(e.PublishedUtc),
|
||||
};
|
||||
|
||||
if (e.LastSuccessfulReadUtc is not null)
|
||||
msg.LastSuccessfulReadUtc = ToUtcTimestamp(e.LastSuccessfulReadUtc.Value);
|
||||
if (e.LastError is not null)
|
||||
msg.LastError = e.LastError;
|
||||
|
||||
return msg;
|
||||
}
|
||||
|
||||
private static DriverResilienceStatus MapResilience(DriverResilienceStatusChanged e)
|
||||
{
|
||||
var msg = new DriverResilienceStatus
|
||||
{
|
||||
DriverInstanceId = e.DriverInstanceId,
|
||||
HostName = e.HostName,
|
||||
BreakerOpen = e.BreakerOpen,
|
||||
ConsecutiveFailures = e.ConsecutiveFailures,
|
||||
CurrentInFlight = e.CurrentInFlight,
|
||||
LastSampledUtc = ToUtcTimestamp(e.LastSampledUtc),
|
||||
PublishedUtc = ToUtcTimestamp(e.PublishedUtc),
|
||||
};
|
||||
|
||||
if (e.LastBreakerOpenUtc is not null)
|
||||
msg.LastBreakerOpenUtc = ToUtcTimestamp(e.LastBreakerOpenUtc.Value);
|
||||
|
||||
return msg;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts a domain <see cref="DateTime"/> to a proto <see cref="Timestamp"/> without ever
|
||||
/// throwing on Kind: Utc is used as-is, Local is converted, and Unspecified is assumed
|
||||
/// already-UTC (the producer contract — <see cref="DateTime.UtcNow"/> — with the wall-clock
|
||||
/// ticks preserved so the instant round-trips).
|
||||
/// </summary>
|
||||
private static Timestamp ToUtcTimestamp(DateTime dt)
|
||||
{
|
||||
var utc = dt.Kind switch
|
||||
{
|
||||
DateTimeKind.Utc => dt,
|
||||
DateTimeKind.Local => dt.ToUniversalTime(),
|
||||
_ => DateTime.SpecifyKind(dt, DateTimeKind.Utc),
|
||||
};
|
||||
|
||||
return Timestamp.FromDateTime(utc);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
|
||||
// The generated service container is itself named TelemetryStreamService; alias its nested server
|
||||
// base so this impl can keep the natural name without colliding with the generated type.
|
||||
using GeneratedServiceBase =
|
||||
ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1.TelemetryStreamService.TelemetryStreamServiceBase;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Node-side live-telemetry streaming server (per-cluster mesh Phase 5). Central dials a driver
|
||||
/// node and opens <see cref="Subscribe"/>; the node attaches a fresh reader on its
|
||||
/// <see cref="ITelemetryLocalHub"/> and pumps this node's own telemetry
|
||||
/// (alarm-transitions / script-logs / driver-health / driver-resilience) to the caller as a
|
||||
/// sequence of <see cref="TelemetryEvent"/> envelopes until the client cancels or the
|
||||
/// max-lifetime cap fires.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The hub carries ONLY this node's telemetry, so a per-node stream never double-counts a peer's
|
||||
/// events. Fan-out is lossy under backpressure by design (the per-subscriber channel is
|
||||
/// bounded/DropOldest inside the hub). A simple process-wide concurrency cap sheds excess dials
|
||||
/// with <see cref="StatusCode.ResourceExhausted"/>, and every accepted stream is bounded by a
|
||||
/// linked max-lifetime CTS.
|
||||
/// </remarks>
|
||||
public sealed class TelemetryStreamGrpcService : GeneratedServiceBase
|
||||
{
|
||||
/// <summary>Process-wide cap on concurrently-open telemetry streams.</summary>
|
||||
private const int MaxConcurrentStreams = 100;
|
||||
|
||||
/// <summary>Upper bound on an accepted correlation id (defends against absurd inputs).</summary>
|
||||
private const int MaxCorrelationIdLength = 256;
|
||||
|
||||
/// <summary>Per-subscriber bounded-channel capacity requested from the hub.</summary>
|
||||
private const int SubscriptionCapacity = 1000;
|
||||
|
||||
/// <summary>Hard ceiling on a single stream's lifetime; the linked CTS cancels the pump when it fires.</summary>
|
||||
private static readonly TimeSpan MaxStreamLifetime = TimeSpan.FromHours(4);
|
||||
|
||||
/// <summary>Process-wide count of currently-open streams; mutated only via <see cref="Interlocked"/>.</summary>
|
||||
private static int _activeStreams;
|
||||
|
||||
private readonly ITelemetryLocalHub _hub;
|
||||
private readonly ILogger<TelemetryStreamGrpcService> _log;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="TelemetryStreamGrpcService"/> class.</summary>
|
||||
/// <param name="hub">The node-local telemetry fan-out hub this stream drains.</param>
|
||||
/// <param name="log">Logger.</param>
|
||||
public TelemetryStreamGrpcService(ITelemetryLocalHub hub, ILogger<TelemetryStreamGrpcService> log)
|
||||
{
|
||||
_hub = hub;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task Subscribe(
|
||||
TelemetryStreamRequest request,
|
||||
IServerStreamWriter<TelemetryEvent> responseStream,
|
||||
ServerCallContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentNullException.ThrowIfNull(responseStream);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
var correlationId = request.CorrelationId;
|
||||
if (string.IsNullOrWhiteSpace(correlationId) || correlationId.Length > MaxCorrelationIdLength)
|
||||
throw new RpcException(new Status(
|
||||
StatusCode.InvalidArgument,
|
||||
"correlation_id must be non-empty and at most 256 characters"));
|
||||
|
||||
// Concurrency cap. The increment is ALWAYS paired: on rejection we decrement here and throw
|
||||
// before entering the try; on acceptance the finally decrements exactly once. No path
|
||||
// decrements without a matching successful increment, and no path double-decrements.
|
||||
var active = Interlocked.Increment(ref _activeStreams);
|
||||
if (active > MaxConcurrentStreams)
|
||||
{
|
||||
Interlocked.Decrement(ref _activeStreams);
|
||||
_log.LogWarning(
|
||||
"Telemetry stream rejected (correlationId={CorrelationId}): {Active} open streams exceeds cap {Cap}",
|
||||
correlationId, active - 1, MaxConcurrentStreams);
|
||||
throw new RpcException(new Status(
|
||||
StatusCode.ResourceExhausted, "too many concurrent telemetry streams"));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var lifetime = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken);
|
||||
lifetime.CancelAfter(MaxStreamLifetime);
|
||||
using var sub = _hub.Subscribe(SubscriptionCapacity);
|
||||
|
||||
_log.LogDebug("Telemetry stream opened (correlationId={CorrelationId})", correlationId);
|
||||
|
||||
try
|
||||
{
|
||||
await foreach (var item in sub.Reader.ReadAllAsync(lifetime.Token).ConfigureAwait(false))
|
||||
await responseStream
|
||||
.WriteAsync(TelemetryProtoMapNode.ToProto(item, correlationId), lifetime.Token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Normal termination: the client disconnected (context token) or the max-lifetime cap
|
||||
// fired (linked token). Neither is an error — swallow and let the stream close cleanly.
|
||||
}
|
||||
|
||||
_log.LogDebug("Telemetry stream closed (correlationId={CorrelationId})", correlationId);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Decrement(ref _activeStreams);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user