dohertj2/lmxopcua
1
0
Fork 0
Code Issues 12 Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

fix(mesh-phase5): harden telemetry client onError contract (validate/isolate/defend)

Browse Source 
Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
2026-07-23 16:12:02 -04:00
parent f131c1cca5
commit a279a43ed4
4 changed files with 258 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Telemetry/TelemetryProtoMapCentral.cs
+31 -6
View File
@@ -1,3 +1,4 @@
using Google.Protobuf.WellKnownTypes;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
@@ -18,10 +19,18 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Telemetry;
/// generated property is <see langword="null"/>, so the nullable domain fields map via
/// <c>?.ToDateTime()</c> and an absent Timestamp becomes <see langword="null"/>. The
/// non-nullable domain fields (published / sampled / transition timestamps) are always set by
/// the node mapper, so they map via a direct <c>.ToDateTime()</c> (which yields a
/// the node mapper, so they map via <see cref="Required"/> (which yields a
/// <see cref="DateTimeKind.Utc"/> value).
/// </para>
/// <para>
/// <b>A required Timestamp is defended, not assumed.</b> Across a mixed-version fleet an
/// out-of-contract sender could leave a required Timestamp unset. <see cref="Required"/> throws
/// a clear <see cref="InvalidOperationException"/> naming the kind + field instead of letting a
/// bare <see cref="NullReferenceException"/> escape — the client's per-event isolation then logs
/// it as a dropped malformed event and continues, because it is a code/version fault, never a
/// transport fault.
/// </para>
/// <para>
/// <b>optional-string presence is honoured.</b> A proto3 <c>optional string</c> that the node
/// left unset reads back as <see langword="null"/> (via the generated <c>Has…</c> presence),
/// distinguishing null from the empty string; likewise the <c>optional bool</c>
@@ -71,7 +80,7 @@ public static class TelemetryProtoMapCentral
Severity: msg.Severity,
Message: msg.Message,
User: msg.User,
TimestampUtc: msg.TimestampUtc.ToDateTime(),
TimestampUtc: Required(msg.TimestampUtc, "AlarmTransition", "timestamp_utc"),
AlarmTypeName: msg.AlarmTypeName,
Comment: msg.HasComment ? msg.Comment : null,
HistorizeToAveva: msg.HasHistorizeToAveva ? msg.HistorizeToAveva : null,
@@ -89,7 +98,7 @@ public static class TelemetryProtoMapCentral
ScriptId: msg.ScriptId,
Level: msg.Level,
Message: msg.Message,
TimestampUtc: msg.TimestampUtc.ToDateTime(),
TimestampUtc: Required(msg.TimestampUtc, "ScriptLog", "timestamp_utc"),
VirtualTagId: msg.HasVirtualTagId ? msg.VirtualTagId : null,
AlarmId: msg.HasAlarmId ? msg.AlarmId : null,
EquipmentId: msg.HasEquipmentId ? msg.EquipmentId : null);
@@ -109,7 +118,7 @@ public static class TelemetryProtoMapCentral
LastSuccessfulReadUtc: msg.LastSuccessfulReadUtc?.ToDateTime(),
LastError: msg.HasLastError ? msg.LastError : null,
ErrorCount5Min: msg.ErrorCount5Min,
PublishedUtc: msg.PublishedUtc.ToDateTime());
PublishedUtc: Required(msg.PublishedUtc, "DriverHealth", "published_utc"));
}
/// <summary>Projects a <see cref="DriverResilienceStatus"/> onto a <see cref="DriverResilienceStatusChanged"/>.</summary>
@@ -126,7 +135,23 @@ public static class TelemetryProtoMapCentral
ConsecutiveFailures: msg.ConsecutiveFailures,
CurrentInFlight: msg.CurrentInFlight,
LastBreakerOpenUtc: msg.LastBreakerOpenUtc?.ToDateTime(),
LastSampledUtc: msg.LastSampledUtc.ToDateTime(),
PublishedUtc: msg.PublishedUtc.ToDateTime());
LastSampledUtc: Required(msg.LastSampledUtc, "DriverResilienceStatus", "last_sampled_utc"),
PublishedUtc: Required(msg.PublishedUtc, "DriverResilienceStatus", "published_utc"));
}
/// <summary>
/// Converts a <b>required</b> proto <see cref="Timestamp"/> to a UTC <see cref="DateTime"/>,
/// throwing a clear <see cref="InvalidOperationException"/> naming the telemetry kind + proto
/// field if an out-of-contract sender left it unset — rather than letting a bare
/// <see cref="NullReferenceException"/> escape.
/// </summary>
/// <param name="ts">The required proto Timestamp (<see langword="null"/> when the sender omitted it).</param>
/// <param name="kind">The telemetry sub-message name, for the diagnostic.</param>
/// <param name="field">The proto field name, for the diagnostic.</param>
/// <returns>The Timestamp as a <see cref="DateTimeKind.Utc"/> value.</returns>
/// <exception cref="InvalidOperationException"><paramref name="ts"/> is unset.</exception>
private static DateTime Required(Timestamp? ts, string kind, string field) =>
ts is null
? throw new InvalidOperationException($"telemetry {kind} missing required timestamp {field}")
: ts.ToDateTime();
}
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Telemetry/TelemetryStreamClient.cs
+71 -14
View File
@@ -1,6 +1,7 @@
using System.Net;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Telemetry;
@@ -14,11 +15,31 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Telemetry;
/// </summary>
/// <remarks>
/// <para>
/// <b>Deliberately dumb.</b> Connect, pump, surface errors — no reconnect, no backoff, no
/// buffering. A normal shutdown (the caller's <see cref="CancellationToken"/> firing, or the
/// server ending the stream with <see cref="StatusCode.Cancelled"/>) completes
/// <see cref="RunAsync"/> without calling <c>onError</c>; any other failure is routed to
/// <c>onError</c> exactly once as the reconnect trigger the supervisor (a later task) owns.
/// <b>Deliberately dumb.</b> Connect, pump, surface transport errors — no reconnect, no
/// backoff, no buffering. A normal shutdown (the caller's <see cref="CancellationToken"/>
/// firing, or the server ending the stream with <see cref="StatusCode.Cancelled"/>) completes
/// <see cref="RunAsync"/> without calling <c>onError</c>.
/// </para>
/// <para>
/// <b>onError is strictly the transport-failure signal.</b> Only a failure from the wire itself
/// — an exception thrown out of <c>call.ResponseStream</c> — reaches <c>onError</c>, and exactly
/// once, so the supervisor's reconnect loop reacts only to a genuinely unreachable/faulted node.
/// Everything that is NOT a transport fault is kept off that path:
/// <list type="bullet">
/// <item>
/// <b>Programming faults</b> (null/empty <c>correlationId</c>, use-after-dispose) are
/// validated <em>before</em> the pump and throw synchronously out of
/// <see cref="RunAsync"/> — a caller bug must surface as a caller bug, never as "node
/// down".
/// </item>
/// <item>
/// <b>Consumer-callback faults</b> (a poison/unmappable event throwing inside
/// <c>onEvent</c> — e.g. a future oneof case the mapper rejects, or a malformed required
/// Timestamp) are caught <em>per event</em>, logged at Warning, and the pump
/// <em>continues</em>. A code/version defect on one event must not tear down the stream
/// or trigger a reconnect.
/// </item>
/// </list>
/// </para>
/// <para>
/// <b>h2c + Bearer.</b> The endpoint is a prior-knowledge <c>http://host:port</c> address (no
@@ -32,6 +53,8 @@ public sealed class TelemetryStreamClient : IDisposable
private readonly string _apiKey;
private readonly GrpcChannel? _ownedChannel;
private readonly TelemetryStreamService.TelemetryStreamServiceClient _client;
private readonly ILogger<TelemetryStreamClient> _log;
private bool _disposed;
/// <summary>
/// Initializes a new instance of the <see cref="TelemetryStreamClient"/> class dialing a live
@@ -39,12 +62,14 @@ public sealed class TelemetryStreamClient : IDisposable
/// </summary>
/// <param name="endpoint">Prior-knowledge <c>http://host:port</c> address of the node's telemetry server.</param>
/// <param name="apiKey">Shared node bearer key sent as <c>authorization: Bearer &lt;key&gt;</c>.</param>
public TelemetryStreamClient(string endpoint, string apiKey)
/// <param name="log">Logger for dropped-malformed-event diagnostics; defaults to a no-op logger.</param>
public TelemetryStreamClient(string endpoint, string apiKey, ILogger<TelemetryStreamClient>? log = null)
{
ArgumentException.ThrowIfNullOrEmpty(endpoint);
ArgumentNullException.ThrowIfNull(apiKey);
_apiKey = apiKey;
_log = log ?? NullLogger<TelemetryStreamClient>.Instance;
_ownedChannel = GrpcChannel.ForAddress(endpoint, new GrpcChannelOptions
{
HttpHandler = new SocketsHttpHandler
@@ -64,34 +89,47 @@ public sealed class TelemetryStreamClient : IDisposable
/// </summary>
/// <param name="client">The generated client to drive (typically a fake in tests).</param>
/// <param name="apiKey">Shared node bearer key sent as <c>authorization: Bearer &lt;key&gt;</c>.</param>
public TelemetryStreamClient(TelemetryStreamService.TelemetryStreamServiceClient client, string apiKey)
/// <param name="log">Logger for dropped-malformed-event diagnostics; defaults to a no-op logger.</param>
public TelemetryStreamClient(
TelemetryStreamService.TelemetryStreamServiceClient client,
string apiKey,
ILogger<TelemetryStreamClient>? log = null)
{
ArgumentNullException.ThrowIfNull(client);
ArgumentNullException.ThrowIfNull(apiKey);
_client = client;
_apiKey = apiKey;
_log = log ?? NullLogger<TelemetryStreamClient>.Instance;
_ownedChannel = null;
}
/// <summary>
/// Opens the telemetry stream and pumps every envelope to <paramref name="onEvent"/> until the
/// server ends the stream or <paramref name="ct"/> fires. A normal shutdown returns without
/// invoking <paramref name="onError"/>; any other failure invokes <paramref name="onError"/>
/// exactly once and returns.
/// invoking <paramref name="onError"/>; only a transport failure invokes
/// <paramref name="onError"/> (exactly once). A consumer-callback exception on one event is
/// logged and the pump continues; programming faults throw synchronously (see the type remarks).
/// </summary>
/// <param name="correlationId">Stream correlation id echoed on every envelope.</param>
/// <param name="onEvent">Sink for each received raw <see cref="TelemetryEvent"/>.</param>
/// <param name="onError">Invoked once on an abnormal stream failure (the reconnect trigger).</param>
/// <param name="onError">Invoked once on an abnormal transport failure (the reconnect trigger).</param>
/// <param name="ct">Cancels the stream (normal shutdown).</param>
/// <exception cref="ArgumentException"><paramref name="correlationId"/> is null or empty.</exception>
/// <exception cref="ArgumentNullException"><paramref name="onEvent"/> or <paramref name="onError"/> is null.</exception>
/// <exception cref="ObjectDisposedException">This client has been disposed.</exception>
public async Task RunAsync(
string correlationId,
Action<TelemetryEvent> onEvent,
Action<Exception> onError,
CancellationToken ct)
{
// Programming faults are validated BEFORE the pump so they throw synchronously out of RunAsync
// rather than funnelling to onError and masquerading as a transport fault (supervisor spin).
ArgumentException.ThrowIfNullOrEmpty(correlationId);
ArgumentNullException.ThrowIfNull(onEvent);
ArgumentNullException.ThrowIfNull(onError);
ObjectDisposedException.ThrowIf(_disposed, this);
var headers = new Metadata { { "authorization", $"Bearer {_apiKey}" } };
@@ -101,7 +139,22 @@ public sealed class TelemetryStreamClient : IDisposable
new TelemetryStreamRequest { CorrelationId = correlationId }, headers, cancellationToken: ct);
await foreach (var evt in call.ResponseStream.ReadAllAsync(ct))
onEvent(evt);
{
// Isolate the consumer callback: a poison/unmappable event is a code/version fault, not
// "node down". Log and continue — never break the stream, never call onError.
try
{
onEvent(evt);
}
catch (Exception ex)
{
_log.LogWarning(
ex,
"Dropping a malformed/unhandled telemetry event on stream {CorrelationId}; " +
"the consumer callback threw. Continuing the stream (not a transport fault).",
correlationId);
}
}
}
catch (OperationCanceledException)
{
@@ -113,11 +166,15 @@ public sealed class TelemetryStreamClient : IDisposable
}
catch (Exception ex)
{
// The reconnect trigger — a later supervisor task decides retry/backoff.
// The transport-failure signal — a later supervisor task decides retry/backoff.
onError(ex);
}
}
/// <inheritdoc />
public void Dispose() => _ownedChannel?.Dispose();
public void Dispose()
{
_disposed = true;
_ownedChannel?.Dispose();
}
}