From 1f3343e61f18540aaf7daeb9863ca9db308b7f20 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 19 Apr 2026 23:16:09 -0400 Subject: [PATCH] =?UTF-8?q?OpenTelemetry=20redundancy=20metrics=20+=20Role?= =?UTF-8?q?Changed=20SignalR=20push.=20Closes=20instrumentation=20+=20live?= =?UTF-8?q?-push=20slices=20of=20task=20#198;=20the=20exporter=20wiring=20?= =?UTF-8?q?(OTLP=20vs=20Prometheus=20package=20decision)=20is=20split=20to?= =?UTF-8?q?=20new=20task=20#201=20because=20the=20collector/scrape-endpoin?= =?UTF-8?q?t=20choice=20is=20a=20fleet-ops=20decision=20that=20deserves=20?= =?UTF-8?q?its=20own=20PR=20rather=20than=20hardcoded=20here.=20New=20Redu?= =?UTF-8?q?ndancyMetrics=20class=20(Singleton-registered=20in=20DI)=20owni?= =?UTF-8?q?ng=20a=20System.Diagnostics.Metrics.Meter("ZB.MOM.WW.OtOpcUa.Re?= =?UTF-8?q?dundancy",=20"1.0.0").=20Three=20ObservableGauge=20instruments?= =?UTF-8?q?=20=E2=80=94=20otopcua.redundancy.primary=5Fcount=20/=20seconda?= =?UTF-8?q?ry=5Fcount=20/=20stale=5Fcount=20=E2=80=94=20all=20tagged=20by?= =?UTF-8?q?=20cluster.id,=20populated=20by=20SetClusterCounts(clusterId,?= =?UTF-8?q?=20primary,=20secondary,=20stale)=20which=20the=20poller=20call?= =?UTF-8?q?s=20at=20the=20tail=20of=20every=20tick;=20ObservableGauge=20ca?= =?UTF-8?q?llbacks=20snapshot=20the=20last=20value=20set=20under=20a=20loc?= =?UTF-8?q?k=20so=20the=20reader=20(OTel=20collector,=20dotnet-counters)?= =?UTF-8?q?=20sees=20consistent=20tuples.=20One=20Counter=20=E2=80=94=20ot?= =?UTF-8?q?opcua.redundancy.role=5Ftransition=20=E2=80=94=20tagged=20clust?= =?UTF-8?q?er.id,=20node.id,=20from=5Frole,=20to=5Frole;=20ideal=20for=20t?= =?UTF-8?q?racking=20"how=20often=20does=20Cluster-X=20failover"=20+=20"wh?= =?UTF-8?q?ich=20node=20transitions=20most"=20aggregate=20queries.=20In-bo?= =?UTF-8?q?x=20Metrics=20API=20means=20zero=20NuGet=20dep=20here=20?= =?UTF-8?q?=E2=80=94=20the=20exporter=20PR=20adds=20OpenTelemetry.Extensio?= =?UTF-8?q?ns.Hosting=20+=20OpenTelemetry.Exporter.OpenTelemetryProtocol?= =?UTF-8?q?=20or=20OpenTelemetry.Exporter.Prometheus.AspNetCore=20to=20act?= =?UTF-8?q?ually=20ship=20the=20data=20somewhere.=20FleetStatusPoller=20ex?= =?UTF-8?q?tended=20with=20role-change=20detection.=20Its=20PollOnceAsync?= =?UTF-8?q?=20now=20pulls=20ClusterNode=20rows=20alongside=20the=20existin?= =?UTF-8?q?g=20ClusterNodeGenerationState=20scan,=20and=20a=20new=20PollRo?= =?UTF-8?q?lesAsync=20walks=20every=20node=20comparing=20RedundancyRole=20?= =?UTF-8?q?to=20the=20=5FlastRole=20cache.=20On=20change:=20records=20the?= =?UTF-8?q?=20transition=20to=20RedundancyMetrics=20+=20emits=20a=20RoleCh?= =?UTF-8?q?anged=20SignalR=20message=20to=20both=20FleetStatusHub.GroupNam?= =?UTF-8?q?e(cluster)=20+=20FleetStatusHub.FleetGroup=20so=20cluster-scope?= =?UTF-8?q?d=20+=20fleet-wide=20subscribers=20both=20see=20it.=20First=20o?= =?UTF-8?q?bservation=20per=20node=20is=20a=20bootstrap=20(cache=20fill)?= =?UTF-8?q?=20+=20NOT=20a=20transition=20=E2=80=94=20avoids=20spurious=20c?= =?UTF-8?q?hurn=20on=20service=20startup=20or=20pod=20restart.=20UpdateClu?= =?UTF-8?q?sterGauges=20groups=20nodes=20by=20cluster=20+=20sets=20the=20t?= =?UTF-8?q?hree=20gauge=20values,=20using=20ClusterNodeService.StaleThresh?= =?UTF-8?q?old=20(shared=2030s=20convention)=20for=20staleness=20so=20the?= =?UTF-8?q?=20/hosts=20page=20+=20the=20gauge=20agree.=20RoleChangedMessag?= =?UTF-8?q?e=20record=20lives=20alongside=20NodeStateChangedMessage=20in?= =?UTF-8?q?=20FleetStatusPoller.cs.=20RedundancyTab.razor=20subscribes=20t?= =?UTF-8?q?o=20the=20fleet-status=20hub=20on=20first=20parameters-set,=20f?= =?UTF-8?q?ilters=20RoleChanged=20events=20to=20the=20current=20cluster,?= =?UTF-8?q?=20reloads=20the=20node=20list=20+=20paints=20a=20blue=20info?= =?UTF-8?q?=20banner=20("Role=20changed=20on=20node-a:=20Primary=20?= =?UTF-8?q?=E2=86=92=20Secondary=20at=20HH:mm:ss=20UTC")=20so=20operators?= =?UTF-8?q?=20see=20the=20transition=20without=20needing=20to=20poll-refre?= =?UTF-8?q?sh=20the=20page.=20IAsyncDisposable=20closes=20the=20connection?= =?UTF-8?q?=20on=20tab=20swap-away.=20Two=20new=20RedundancyMetricsTests?= =?UTF-8?q?=20covering=20RecordRoleTransition=20tag=20emission=20(cluster.?= =?UTF-8?q?id=20+=20node.id=20+=20from=5Frole=20+=20to=5Frole=20all=20flow?= =?UTF-8?q?=20through=20the=20MeterListener=20callback)=20+=20ObservableGa?= =?UTF-8?q?uge=20snapshot=20for=20two=20clusters=20(assert=20primary=5Fcou?= =?UTF-8?q?nt=3D1=20for=20c1,=20stale=5Fcount=3D1=20for=20c2).=20Existing?= =?UTF-8?q?=20FleetStatusPollerTests=20ctor-line=20updated=20to=20pass=20a?= =?UTF-8?q?=20RedundancyMetrics=20instance;=20all=20tests=20still=20pass.?= =?UTF-8?q?=20Full=20Admin.Tests=20suite=2087/87=20passing=20(was=2085,=20?= =?UTF-8?q?+2).=20Admin=20project=20builds=200=20errors.=20Task=20#201=20c?= =?UTF-8?q?aptures=20the=20exporter-wiring=20follow-up=20=E2=80=94=20OpenT?= =?UTF-8?q?elemetry.Extensions.Hosting=20+=20OTLP=20vs=20Prometheus=20+=20?= =?UTF-8?q?/metrics=20endpoint=20decision,=20driven=20by=20fleet-ops=20inf?= =?UTF-8?q?ra=20direction.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- .../Pages/Clusters/RedundancyTab.razor | 39 +++++++ .../Hubs/FleetStatusPoller.cs | 66 +++++++++++- src/ZB.MOM.WW.OtOpcUa.Admin/Program.cs | 1 + .../Services/RedundancyMetrics.cs | 102 ++++++++++++++++++ .../FleetStatusPollerTests.cs | 5 +- .../RedundancyMetricsTests.cs | 70 ++++++++++++ 6 files changed, 279 insertions(+), 4 deletions(-) create mode 100644 src/ZB.MOM.WW.OtOpcUa.Admin/Services/RedundancyMetrics.cs create mode 100644 tests/ZB.MOM.WW.OtOpcUa.Admin.Tests/RedundancyMetricsTests.cs diff --git a/src/ZB.MOM.WW.OtOpcUa.Admin/Components/Pages/Clusters/RedundancyTab.razor b/src/ZB.MOM.WW.OtOpcUa.Admin/Components/Pages/Clusters/RedundancyTab.razor index 068f059..c562415 100644 --- a/src/ZB.MOM.WW.OtOpcUa.Admin/Components/Pages/Clusters/RedundancyTab.razor +++ b/src/ZB.MOM.WW.OtOpcUa.Admin/Components/Pages/Clusters/RedundancyTab.razor @@ -1,9 +1,17 @@ +@using Microsoft.AspNetCore.SignalR.Client +@using ZB.MOM.WW.OtOpcUa.Admin.Hubs @using ZB.MOM.WW.OtOpcUa.Admin.Services @using ZB.MOM.WW.OtOpcUa.Configuration.Entities @using ZB.MOM.WW.OtOpcUa.Configuration.Enums @inject ClusterNodeService NodeSvc +@inject NavigationManager Nav +@implements IAsyncDisposable

Redundancy topology

+@if (_roleChangedBanner is not null) +{ +
@_roleChangedBanner
+}

One row per ClusterNode in this cluster. Role, ApplicationUri, and ServiceLevelBase are authored separately; the Admin UI shows them read-only @@ -107,10 +115,41 @@ else [Parameter] public string ClusterId { get; set; } = string.Empty; private List? _nodes; + private HubConnection? _hub; + private string? _roleChangedBanner; protected override async Task OnParametersSetAsync() { _nodes = await NodeSvc.ListByClusterAsync(ClusterId, CancellationToken.None); + if (_hub is null) await ConnectHubAsync(); + } + + private async Task ConnectHubAsync() + { + _hub = new HubConnectionBuilder() + .WithUrl(Nav.ToAbsoluteUri("/hubs/fleet-status")) + .WithAutomaticReconnect() + .Build(); + + _hub.On("RoleChanged", async msg => + { + if (msg.ClusterId != ClusterId) return; + _roleChangedBanner = $"Role changed on {msg.NodeId}: {msg.FromRole} → {msg.ToRole} at {msg.ObservedAtUtc:HH:mm:ss 'UTC'}"; + _nodes = await NodeSvc.ListByClusterAsync(ClusterId, CancellationToken.None); + await InvokeAsync(StateHasChanged); + }); + + await _hub.StartAsync(); + await _hub.SendAsync("SubscribeCluster", ClusterId); + } + + public async ValueTask DisposeAsync() + { + if (_hub is not null) + { + await _hub.DisposeAsync(); + _hub = null; + } } private static string RowClass(ClusterNode n) => diff --git a/src/ZB.MOM.WW.OtOpcUa.Admin/Hubs/FleetStatusPoller.cs b/src/ZB.MOM.WW.OtOpcUa.Admin/Hubs/FleetStatusPoller.cs index bead926..cd617d1 100644 --- a/src/ZB.MOM.WW.OtOpcUa.Admin/Hubs/FleetStatusPoller.cs +++ b/src/ZB.MOM.WW.OtOpcUa.Admin/Hubs/FleetStatusPoller.cs @@ -1,7 +1,9 @@ using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Admin.Services; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; namespace ZB.MOM.WW.OtOpcUa.Admin.Hubs; @@ -14,11 +16,13 @@ public sealed class FleetStatusPoller( IServiceScopeFactory scopeFactory, IHubContext fleetHub, IHubContext alertHub, - ILogger logger) : BackgroundService + ILogger logger, + RedundancyMetrics redundancyMetrics) : BackgroundService { public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(5); private readonly Dictionary _last = new(); + private readonly Dictionary _lastRole = new(StringComparer.Ordinal); protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -42,6 +46,10 @@ public sealed class FleetStatusPoller( using var scope = scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); + var nodes = await db.ClusterNodes.AsNoTracking().ToListAsync(ct); + await PollRolesAsync(nodes, ct); + UpdateClusterGauges(nodes); + var rows = await db.ClusterNodeGenerationStates.AsNoTracking() .Join(db.ClusterNodes.AsNoTracking(), s => s.NodeId, n => n.NodeId, (s, n) => new { s, n.ClusterId }) .ToListAsync(ct); @@ -85,9 +93,63 @@ public sealed class FleetStatusPoller( } ///

Exposed for tests — forces a snapshot reset so stub data re-seeds. - internal void ResetCache() => _last.Clear(); + internal void ResetCache() + { + _last.Clear(); + _lastRole.Clear(); + } + + private async Task PollRolesAsync(IReadOnlyList nodes, CancellationToken ct) + { + foreach (var n in nodes) + { + var hadPrior = _lastRole.TryGetValue(n.NodeId, out var priorRole); + if (hadPrior && priorRole == n.RedundancyRole) continue; + + _lastRole[n.NodeId] = n.RedundancyRole; + if (!hadPrior) continue; // first-observation bootstrap — not a transition + + redundancyMetrics.RecordRoleTransition( + clusterId: n.ClusterId, nodeId: n.NodeId, + fromRole: priorRole.ToString(), toRole: n.RedundancyRole.ToString()); + + var msg = new RoleChangedMessage( + ClusterId: n.ClusterId, NodeId: n.NodeId, + FromRole: priorRole.ToString(), ToRole: n.RedundancyRole.ToString(), + ObservedAtUtc: DateTime.UtcNow); + + await fleetHub.Clients.Group(FleetStatusHub.GroupName(n.ClusterId)) + .SendAsync("RoleChanged", msg, ct); + await fleetHub.Clients.Group(FleetStatusHub.FleetGroup) + .SendAsync("RoleChanged", msg, ct); + } + } + + private void UpdateClusterGauges(IReadOnlyList nodes) + { + var staleCutoff = DateTime.UtcNow - Services.ClusterNodeService.StaleThreshold; + foreach (var group in nodes.GroupBy(n => n.ClusterId)) + { + var primary = group.Count(n => n.RedundancyRole == RedundancyRole.Primary); + var secondary = group.Count(n => n.RedundancyRole == RedundancyRole.Secondary); + var stale = group.Count(n => n.LastSeenAt is null || n.LastSeenAt.Value < staleCutoff); + redundancyMetrics.SetClusterCounts(group.Key, primary, secondary, stale); + } + } private readonly record struct NodeStateSnapshot( string NodeId, string ClusterId, long? GenerationId, string? Status, string? Error, DateTime? AppliedAt, DateTime? SeenAt); } + +/// +/// Pushed by when it observes a change in +/// . Consumed by the Admin RedundancyTab to trigger +/// an instant reload instead of waiting for the next on-parameter-set poll. +/// +public sealed record RoleChangedMessage( + string ClusterId, + string NodeId, + string FromRole, + string ToRole, + DateTime ObservedAtUtc); diff --git a/src/ZB.MOM.WW.OtOpcUa.Admin/Program.cs b/src/ZB.MOM.WW.OtOpcUa.Admin/Program.cs index a0fe448..b6f3ea3 100644 --- a/src/ZB.MOM.WW.OtOpcUa.Admin/Program.cs +++ b/src/ZB.MOM.WW.OtOpcUa.Admin/Program.cs @@ -49,6 +49,7 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/src/ZB.MOM.WW.OtOpcUa.Admin/Services/RedundancyMetrics.cs b/src/ZB.MOM.WW.OtOpcUa.Admin/Services/RedundancyMetrics.cs new file mode 100644 index 0000000..9def449 --- /dev/null +++ b/src/ZB.MOM.WW.OtOpcUa.Admin/Services/RedundancyMetrics.cs @@ -0,0 +1,102 @@ +using System.Diagnostics.Metrics; + +namespace ZB.MOM.WW.OtOpcUa.Admin.Services; + +/// +/// OpenTelemetry-compatible instrumentation for the redundancy surface. Uses in-box +/// so no NuGet dependency is required to emit — +/// any MeterListener (dotnet-counters, OpenTelemetry.Extensions.Hosting OTLP exporter, +/// Prometheus exporter, etc.) picks up the instruments by the . +/// +/// +/// Exporter configuration (OTLP, Prometheus, etc.) is intentionally NOT wired here — +/// that's a deployment-ops decision that belongs in Program.cs behind an +/// appsettings toggle. This class owns only the Meter + instruments so the +/// production data stream exists regardless of exporter availability. +/// +/// Counter + gauge names follow the otel-semantic-conventions pattern: +/// otopcua.redundancy.* with tags for ClusterId + (for transitions) FromRole/ToRole/NodeId. +/// +public sealed class RedundancyMetrics : IDisposable +{ + public const string MeterName = "ZB.MOM.WW.OtOpcUa.Redundancy"; + + private readonly Meter _meter; + private readonly Counter _roleTransitions; + private readonly object _gaugeLock = new(); + private readonly Dictionary _gaugeState = new(); + + public RedundancyMetrics() + { + _meter = new Meter(MeterName, version: "1.0.0"); + _roleTransitions = _meter.CreateCounter( + "otopcua.redundancy.role_transition", + unit: "{transition}", + description: "Observed RedundancyRole changes per node — tagged FromRole, ToRole, NodeId, ClusterId."); + + // Observable gauges — the callback reports whatever the last Observe*Count call stashed. + _meter.CreateObservableGauge( + "otopcua.redundancy.primary_count", + ObservePrimaryCounts, + unit: "{node}", + description: "Count of Primary-role nodes per cluster (should be 1 for N+1 redundant clusters, 0 during failover)."); + _meter.CreateObservableGauge( + "otopcua.redundancy.secondary_count", + ObserveSecondaryCounts, + unit: "{node}", + description: "Count of Secondary-role nodes per cluster."); + _meter.CreateObservableGauge( + "otopcua.redundancy.stale_count", + ObserveStaleCounts, + unit: "{node}", + description: "Count of cluster nodes whose LastSeenAt is older than StaleThreshold."); + } + + /// + /// Update the per-cluster snapshot consumed by the ObservableGauges. Poller calls this + /// at the end of every tick so the collectors see fresh numbers on the next observation + /// window (by default 1s for dotnet-counters, configurable per exporter). + /// + public void SetClusterCounts(string clusterId, int primary, int secondary, int stale) + { + lock (_gaugeLock) + { + _gaugeState[clusterId] = new ClusterGaugeState(primary, secondary, stale); + } + } + + /// + /// Increment the role_transition counter when a node's RedundancyRole changes. Tags + /// allow breakdowns by from/to roles (e.g. Primary → Secondary for planned failover vs + /// Primary → Standalone for emergency recovery) + by cluster for multi-site fleets. + /// + public void RecordRoleTransition(string clusterId, string nodeId, string fromRole, string toRole) + { + _roleTransitions.Add(1, + new KeyValuePair("cluster.id", clusterId), + new KeyValuePair("node.id", nodeId), + new KeyValuePair("from_role", fromRole), + new KeyValuePair("to_role", toRole)); + } + + public void Dispose() => _meter.Dispose(); + + private IEnumerable> ObservePrimaryCounts() => SnapshotGauge(s => s.Primary); + private IEnumerable> ObserveSecondaryCounts() => SnapshotGauge(s => s.Secondary); + private IEnumerable> ObserveStaleCounts() => SnapshotGauge(s => s.Stale); + + private IEnumerable> SnapshotGauge(Func selector) + { + List> results; + lock (_gaugeLock) + { + results = new List>(_gaugeState.Count); + foreach (var (cluster, state) in _gaugeState) + results.Add(new Measurement(selector(state), + new KeyValuePair("cluster.id", cluster))); + } + return results; + } + + private readonly record struct ClusterGaugeState(int Primary, int Secondary, int Stale); +} diff --git a/tests/ZB.MOM.WW.OtOpcUa.Admin.Tests/FleetStatusPollerTests.cs b/tests/ZB.MOM.WW.OtOpcUa.Admin.Tests/FleetStatusPollerTests.cs index 906388d..df07586 100644 --- a/tests/ZB.MOM.WW.OtOpcUa.Admin.Tests/FleetStatusPollerTests.cs +++ b/tests/ZB.MOM.WW.OtOpcUa.Admin.Tests/FleetStatusPollerTests.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Admin.Hubs; +using ZB.MOM.WW.OtOpcUa.Admin.Services; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; @@ -97,7 +98,7 @@ END"; var poller = new FleetStatusPoller( _sp.GetRequiredService(), - fleetHub, alertHub, NullLogger.Instance); + fleetHub, alertHub, NullLogger.Instance, new RedundancyMetrics()); await poller.PollOnceAsync(CancellationToken.None); @@ -142,7 +143,7 @@ END"; var poller = new FleetStatusPoller( _sp.GetRequiredService(), - fleetHub, alertHub, NullLogger.Instance); + fleetHub, alertHub, NullLogger.Instance, new RedundancyMetrics()); await poller.PollOnceAsync(CancellationToken.None); diff --git a/tests/ZB.MOM.WW.OtOpcUa.Admin.Tests/RedundancyMetricsTests.cs b/tests/ZB.MOM.WW.OtOpcUa.Admin.Tests/RedundancyMetricsTests.cs new file mode 100644 index 0000000..4faee99 --- /dev/null +++ b/tests/ZB.MOM.WW.OtOpcUa.Admin.Tests/RedundancyMetricsTests.cs @@ -0,0 +1,70 @@ +using System.Diagnostics.Metrics; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Admin.Services; + +namespace ZB.MOM.WW.OtOpcUa.Admin.Tests; + +[Trait("Category", "Unit")] +public sealed class RedundancyMetricsTests +{ + [Fact] + public void RecordRoleTransition_Increments_Counter_WithExpectedTags() + { + using var metrics = new RedundancyMetrics(); + using var listener = new MeterListener(); + var observed = new List<(long Value, Dictionary Tags)>(); + listener.InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name == RedundancyMetrics.MeterName && + instrument.Name == "otopcua.redundancy.role_transition") + { + l.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((_, value, tags, _) => + { + var dict = new Dictionary(); + foreach (var tag in tags) dict[tag.Key] = tag.Value; + observed.Add((value, dict)); + }); + listener.Start(); + + metrics.RecordRoleTransition("c1", "node-a", "Primary", "Secondary"); + + observed.Count.ShouldBe(1); + observed[0].Value.ShouldBe(1); + observed[0].Tags["cluster.id"].ShouldBe("c1"); + observed[0].Tags["node.id"].ShouldBe("node-a"); + observed[0].Tags["from_role"].ShouldBe("Primary"); + observed[0].Tags["to_role"].ShouldBe("Secondary"); + } + + [Fact] + public void SetClusterCounts_Observed_Via_ObservableGauges() + { + using var metrics = new RedundancyMetrics(); + metrics.SetClusterCounts("c1", primary: 1, secondary: 2, stale: 0); + metrics.SetClusterCounts("c2", primary: 0, secondary: 1, stale: 1); + + var observations = new List<(string Name, long Value, string Cluster)>(); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name == RedundancyMetrics.MeterName) + l.EnableMeasurementEvents(instrument); + }; + listener.SetMeasurementEventCallback((instrument, value, tags, _) => + { + string? cluster = null; + foreach (var t in tags) if (t.Key == "cluster.id") cluster = t.Value as string; + observations.Add((instrument.Name, value, cluster ?? "?")); + }); + listener.Start(); + listener.RecordObservableInstruments(); + + observations.ShouldContain(o => o.Name == "otopcua.redundancy.primary_count" && o.Cluster == "c1" && o.Value == 1); + observations.ShouldContain(o => o.Name == "otopcua.redundancy.secondary_count" && o.Cluster == "c1" && o.Value == 2); + observations.ShouldContain(o => o.Name == "otopcua.redundancy.stale_count" && o.Cluster == "c2" && o.Value == 1); + } +} -- 2.49.1