feat(mesh-phase5): host the telemetry gRPC server on driver nodes (dedicated h2c port)

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
2026-07-23 16:05:40 -04:00
parent 2e70ef88aa
commit f131c1cca5
+29 -13
View File
@@ -22,6 +22,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Host; using ZB.MOM.WW.OtOpcUa.Host;
using ZB.MOM.WW.OtOpcUa.Host.Configuration; using ZB.MOM.WW.OtOpcUa.Host.Configuration;
using ZB.MOM.WW.OtOpcUa.Host.Drivers; using ZB.MOM.WW.OtOpcUa.Host.Drivers;
using ZB.MOM.WW.OtOpcUa.Host.Grpc;
using ZB.MOM.WW.OtOpcUa.Host.Engines; using ZB.MOM.WW.OtOpcUa.Host.Engines;
using ZB.MOM.WW.OtOpcUa.Host.Health; using ZB.MOM.WW.OtOpcUa.Host.Health;
using ZB.MOM.WW.OtOpcUa.Host.Logging; using ZB.MOM.WW.OtOpcUa.Host.Logging;
@@ -429,33 +430,37 @@ if (hasDriver || hasAdmin)
{ {
if (hasDriver) if (hasDriver)
o.Interceptors.Add<LocalDbSyncAuthInterceptor>(); o.Interceptors.Add<LocalDbSyncAuthInterceptor>();
if (hasDriver)
o.Interceptors.Add<TelemetryStreamAuthInterceptor>();
if (hasAdmin) if (hasAdmin)
o.Interceptors.Add<ConfigServeAuthInterceptor>(); o.Interceptors.Add<ConfigServeAuthInterceptor>();
}); });
} }
// --------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------
// Dedicated h2c listeners (both default-OFF): the LocalDb sync endpoint (driver) and the // Dedicated h2c listeners (all default-OFF): the LocalDb sync endpoint (driver), the ConfigServe
// ConfigServe artifact endpoint (admin, per-cluster mesh Phase 3). // artifact endpoint (admin, per-cluster mesh Phase 3), and the telemetry-stream endpoint (driver,
// per-cluster mesh Phase 5).
// //
// DANGER: any explicit Kestrel Listen* call makes Kestrel IGNORE ASPNETCORE_URLS/urls ENTIRELY // DANGER: any explicit Kestrel Listen* call makes Kestrel IGNORE ASPNETCORE_URLS/urls ENTIRELY
// (it logs "Overriding address(es)"). The host has no ConfigureKestrel today and binds solely via // (it logs "Overriding address(es)"). The host has no ConfigureKestrel today and binds solely via
// that configuration, so adding a listener naively would silently unbind the AdminUI + deploy API // that configuration, so adding a listener naively would silently unbind the AdminUI + deploy API
// behind Traefik. Everything the host was already asked to serve is therefore re-bound explicitly // behind Traefik. Everything the host was already asked to serve is therefore re-bound explicitly
// in the same block — and, critically, EXACTLY ONCE: a fused admin+driver node can have BOTH // in the same block — and, critically, EXACTLY ONCE: a fused admin+driver node can have ALL THREE
// dedicated ports set, and re-applying the existing surface per-port would double-bind it and throw // dedicated ports set, and re-applying the existing surface per-port would double-bind it and throw
// "address already in use". So the existing surface is computed and applied once, then whichever of // "address already in use". So the existing surface is computed and applied once, then whichever of
// the two dedicated ports are configured are added on top. // the three dedicated ports are configured are added on top.
// //
// Each listener is HTTP/2-ONLY on purpose: both clients speak prior-knowledge h2c, which a cleartext // Each listener is HTTP/2-ONLY on purpose: every client speaks prior-knowledge h2c, which a cleartext
// Http1AndHttp2 endpoint cannot negotiate (there is no ALPN without TLS). Hence dedicated ports // Http1AndHttp2 endpoint cannot negotiate (there is no ALPN without TLS). Hence dedicated ports
// rather than multiplexing onto the main one. // rather than multiplexing onto the main one.
// //
// When both ports are 0 (the default) none of this runs and URL binding is untouched. // When all three ports are 0 (the default) none of this runs and URL binding is untouched.
// --------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------
var syncListenPort = hasDriver ? LocalDbRegistration.SyncListenPort(builder.Configuration) : 0; var syncListenPort = hasDriver ? LocalDbRegistration.SyncListenPort(builder.Configuration) : 0;
var configServeGrpcPort = hasAdmin ? builder.Configuration.GetValue<int>("ConfigServe:GrpcListenPort") : 0; var configServeGrpcPort = hasAdmin ? builder.Configuration.GetValue<int>("ConfigServe:GrpcListenPort") : 0;
if (syncListenPort > 0 || configServeGrpcPort > 0) var telemetryListenPort = hasDriver ? builder.Configuration.GetValue<int>("Telemetry:GrpcListenPort") : 0;
if (syncListenPort > 0 || configServeGrpcPort > 0 || telemetryListenPort > 0)
{ {
// Mirror Kestrel's own source precedence: URLS wins; else the HTTP_PORTS/HTTPS_PORTS bare-port // Mirror Kestrel's own source precedence: URLS wins; else the HTTP_PORTS/HTTPS_PORTS bare-port
// vars; else Kestrel's localhost:5000 default. Missing the HTTP_PORTS leg is not academic — the // vars; else Kestrel's localhost:5000 default. Missing the HTTP_PORTS leg is not academic — the
@@ -504,19 +509,22 @@ if (syncListenPort > 0 || configServeGrpcPort > 0)
{ {
Log.Error( Log.Error(
"A dedicated h2c listener is requested (LocalDb:SyncListenPort={SyncPort}, " + "A dedicated h2c listener is requested (LocalDb:SyncListenPort={SyncPort}, " +
"ConfigServe:GrpcListenPort={ConfigServePort}) but this host serves HTTPS endpoint(s) ({Urls}). " + "ConfigServe:GrpcListenPort={ConfigServePort}, Telemetry:GrpcListenPort={TelemetryPort}) but this " +
"host serves HTTPS endpoint(s) ({Urls}). " +
"Binding a dedicated listener requires re-binding every existing endpoint explicitly, and the " + "Binding a dedicated listener requires re-binding every existing endpoint explicitly, and the " +
"HTTPS certificate configuration cannot be replayed safely. BOTH dedicated listeners are DISABLED; " + "HTTPS certificate configuration cannot be replayed safely. ALL dedicated listeners are DISABLED; " +
"terminate TLS upstream (as the docker-dev rig does) or leave these features off on this node.", "terminate TLS upstream (as the docker-dev rig does) or leave these features off on this node.",
syncListenPort, configServeGrpcPort, configuredUrls); syncListenPort, configServeGrpcPort, telemetryListenPort, configuredUrls);
syncListenPort = 0; syncListenPort = 0;
configServeGrpcPort = 0; configServeGrpcPort = 0;
telemetryListenPort = 0;
} }
else else
{ {
// Capture locals so the ConfigureKestrel closure does not observe later reassignment. // Capture locals so the ConfigureKestrel closure does not observe later reassignment.
var syncPortToBind = syncListenPort; var syncPortToBind = syncListenPort;
var configServePortToBind = configServeGrpcPort; var configServePortToBind = configServeGrpcPort;
var telemetryPortToBind = telemetryListenPort;
builder.WebHost.ConfigureKestrel(kestrel => builder.WebHost.ConfigureKestrel(kestrel =>
{ {
foreach (var binding in existingBindings) foreach (var binding in existingBindings)
@@ -526,12 +534,15 @@ if (syncListenPort > 0 || configServeGrpcPort > 0)
kestrel.ListenAnyIP(syncPortToBind, o => o.Protocols = HttpProtocols.Http2); kestrel.ListenAnyIP(syncPortToBind, o => o.Protocols = HttpProtocols.Http2);
if (configServePortToBind > 0) if (configServePortToBind > 0)
kestrel.ListenAnyIP(configServePortToBind, o => o.Protocols = HttpProtocols.Http2); kestrel.ListenAnyIP(configServePortToBind, o => o.Protocols = HttpProtocols.Http2);
if (telemetryPortToBind > 0)
kestrel.ListenAnyIP(telemetryPortToBind, o => o.Protocols = HttpProtocols.Http2);
}); });
Log.Information( Log.Information(
"Dedicated h2c listener(s) bound (sync=:{SyncPort}, config-serve=:{ConfigServePort}); " + "Dedicated h2c listener(s) bound (sync=:{SyncPort}, config-serve=:{ConfigServePort}, " +
"re-bound existing endpoint(s) {Urls}.", "telemetry=:{TelemetryPort}); re-bound existing endpoint(s) {Urls}.",
syncListenPort, configServeGrpcPort, configuredUrls ?? "http://localhost:5000 (Kestrel default)"); syncListenPort, configServeGrpcPort, telemetryListenPort,
configuredUrls ?? "http://localhost:5000 (Kestrel default)");
} }
} }
@@ -582,6 +593,11 @@ if (hasAdmin && configServeGrpcPort > 0)
app.MapGrpcService<DeploymentArtifactGrpcService>(); app.MapGrpcService<DeploymentArtifactGrpcService>();
} }
// Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by
// TelemetryStreamAuthInterceptor (fail-closed on empty Telemetry:ApiKey).
if (hasDriver && telemetryListenPort > 0)
app.MapGrpcService<TelemetryStreamGrpcService>();
app.MapOtOpcUaHealth(); app.MapOtOpcUaHealth();
app.MapOtOpcUaMetrics(); app.MapOtOpcUaMetrics();