-
Faulted devices
-
@_snapshot.Drivers.Sum(d => d.FaultedDevices)
+ @* #786: distinct devices in Faulted across every node of the selected node's cluster (a device faulted on
+ both nodes of a redundant pair counts once) — Stopped is not a fault and is a per-node column below.
+ See FleetDiagnosticsAggregation for the identity key. *@
+
+
Faulted devices@(_clusterId is { } cid ? $" (cluster {cid})" : "")
+
@_clusterFaulted
+ @if (_clusterNodeCount > 1)
+ {
+
@_clusterAnswered of @_clusterNodeCount nodes answered
+ }
@@ -124,6 +132,7 @@ else
State |
Connected devices |
Faulted devices |
+
Stopped devices |
Last health publish |
@@ -144,6 +153,7 @@ else
@d.State |
@d.ConnectedDevices |
@d.FaultedDevices |
+
@d.StoppedDevices |
@d.LastChangeUtc.ToString("u") |
}
@@ -165,6 +175,12 @@ else
private NodeDiagnosticsSnapshot? _snapshot;
private IReadOnlyDictionary
_displayNames = new Dictionary();
private bool _loading;
+ // #786 cluster-wide faulted-device tile: the selected node's cluster, how many of its nodes answered, and the
+ // distinct faulted-device count across them.
+ private string? _clusterId;
+ private int _clusterNodeCount;
+ private int _clusterAnswered;
+ private int _clusterFaulted;
private DateTime? _fetchedUtc;
// Page-lifetime cancellation: a fetch in flight when the circuit goes away is abandoned.
@@ -222,6 +238,7 @@ else
// The client is fail-safe by contract: a node that does not answer yields an empty snapshot.
_snapshot = await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(_selectedNode), _cts.Token);
_fetchedUtc = DateTime.UtcNow;
+ await LoadClusterFaultedAsync(_snapshot);
_displayNames = await LoadDisplayNamesAsync(_snapshot);
}
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
@@ -235,6 +252,48 @@ else
}
}
+ // #786: fetch the selected node's cluster peers too and count distinct faulted devices across all of them.
+ // The ConfigDb lookup is enrichment: on failure the tile falls back to the selected node alone.
+ private async Task LoadClusterFaultedAsync(NodeDiagnosticsSnapshot selected)
+ {
+ string? clusterId = null;
+ IReadOnlyList nodeIds = new[] { selected.NodeId.Value };
+ try
+ {
+ await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
+ (clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(db, selected.NodeId.Value, _cts.Token);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
+ {
+ // ConfigDb hiccup — count this node only.
+ }
+
+ var snapshots = new List { selected };
+ var peers = nodeIds.Where(n => !string.Equals(n, selected.NodeId.Value, StringComparison.OrdinalIgnoreCase)).ToList();
+ var peerSnapshots = await Task.WhenAll(peers.Select(FetchPeerAsync));
+ snapshots.AddRange(peerSnapshots.OfType());
+
+ _clusterId = clusterId;
+ _clusterNodeCount = nodeIds.Count;
+ // The client degrades an unreachable node to an empty snapshot; one with drivers clearly answered. An empty
+ // answer is indistinguishable from "no answer", so it is counted as not answered — it contributes nothing either way.
+ _clusterAnswered = 1 + peerSnapshots.Count(p => p is { Drivers.Count: > 0 });
+ _clusterFaulted = FleetDiagnosticsAggregation.DistinctFaultedDevices(snapshots);
+ }
+
+ private async Task FetchPeerAsync(string nodeId)
+ {
+ try
+ {
+ return await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(nodeId), _cts.Token);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
+ {
+ // A malformed peer row (NodeId.Parse) — the peer just does not contribute.
+ return null;
+ }
+ }
+
// The authored name for a driver row, else its id (no ConfigDb row, or the name lookup failed).
private string DisplayNameOf(DriverInstanceDiagnostics d) =>
_displayNames.TryGetValue(d.Name, out var name) ? name : d.Name;
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
index 3d58971c9..254a7ee21 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
@@ -2755,9 +2755,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// One child's diagnostics row. Name carries the string driver-instance id — the identity every
/// other surface (/hosts, telemetry, the artifact) keys on — and DriverInstanceId the DB row Guid;
/// the wire record has no slot for the authored display name. Device counts come from the driver's host
- /// statuses when it is an IHostConnectivityProbe (Running = connected; Stopped/Faulted = faulted;
- /// Unknown = neither); a single-host driver counts as one device, connected while it reports Healthy or
- /// Degraded and faulted while it reports Faulted. A child that has not published yet reports
+ /// statuses when it is an IHostConnectivityProbe (Running = connected; Faulted = faulted;
+ /// Stopped = stopped, counted separately since #786; Unknown = none) and the faulted hosts' names ride
+ /// along so the AdminUI can de-duplicate a device across a redundant pair; a single-host driver counts as
+ /// one device, connected while it reports Healthy or Degraded and faulted while it reports Faulted. A child that has not published yet reports
/// "Connecting" (or "Stubbed") with its spawn time as the last change.
///
private static DriverInstanceDiagnostics ToDiagnostics(string driverInstanceId, ChildEntry entry, ChildHealthRecord? health)
@@ -2768,20 +2769,40 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
if (health is null)
return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Connecting", 0, 0, entry.SpawnedUtc);
- int connected, faulted;
+ return DiagnosticsFromHealth(entry.Spec.DriverInstanceRowId, name, health);
+ }
+
+ /// The device counts of a child that has published health (#786: Faulted and Stopped split).
+ /// The driver instance's ConfigDb row id.
+ /// The string driver-instance id.
+ /// The child's last-published health.
+ /// The diagnostics row.
+ internal static DriverInstanceDiagnostics DiagnosticsFromHealth(Guid rowId, string name, ChildHealthRecord health)
+ {
+ int connected, faulted, stopped;
+ IReadOnlyList faultedHostNames;
if (health.HostStatuses is { Count: > 0 } hosts)
{
connected = hosts.Count(h => h.State == HostState.Running);
- faulted = hosts.Count(h => h.State is HostState.Stopped or HostState.Faulted);
+ // #786: Faulted only. A Stopped host (e.g. a Galaxy engine an operator took off scan) is a
+ // deliberate, separately-reported state, not a fault.
+ var faultedHosts = hosts.Where(h => h.State == HostState.Faulted).ToArray();
+ faulted = faultedHosts.Length;
+ stopped = hosts.Count(h => h.State == HostState.Stopped);
+ faultedHostNames = faultedHosts.Select(h => h.HostName).ToArray();
}
else
{
connected = health.State is DriverState.Healthy or DriverState.Degraded ? 1 : 0;
faulted = health.State == DriverState.Faulted ? 1 : 0;
+ stopped = 0;
+ // A single-host driver is its own device: no host name; the AdminUI keys it on the instance id.
+ faultedHostNames = Array.Empty();
}
return new DriverInstanceDiagnostics(
- entry.Spec.DriverInstanceRowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc);
+ rowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc,
+ StoppedDevices: stopped, FaultedHostNames: faultedHostNames);
}
private void HandleDispatchFromSteady(DispatchDeployment msg)
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/FleetDiagnosticsAggregationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/FleetDiagnosticsAggregationTests.cs
new file mode 100644
index 000000000..0eb62f8e6
--- /dev/null
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/FleetDiagnosticsAggregationTests.cs
@@ -0,0 +1,165 @@
+using System.Text;
+using Akka.Actor;
+using Microsoft.EntityFrameworkCore;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
+using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests;
+
+///
+/// #786: the /fleet/diagnostics "Faulted devices" tile counts DISTINCT faulted devices across a
+/// cluster's nodes (a device faulted on both nodes of a redundant pair counts once), Faulted only.
+///
+[Trait("Category", "Unit")]
+public sealed class FleetDiagnosticsAggregationTests
+{
+ private static readonly DateTime Now = DateTime.UtcNow;
+
+ private static DriverInstanceDiagnostics Driver(
+ string name, int faulted, IReadOnlyList? faultedHosts, int stopped = 0) =>
+ new(Guid.NewGuid(), name, "Degraded", 0, faulted, Now, stopped, faultedHosts);
+
+ private static NodeDiagnosticsSnapshot Node(string nodeId, params DriverInstanceDiagnostics[] drivers) =>
+ new(NodeId.Parse(nodeId), null, drivers, Now);
+
+ [Fact]
+ public void The_same_device_faulted_on_both_nodes_of_a_pair_counts_once()
+ {
+ var a = Node("node-a:4053", Driver("drv-modbus", 2, new[] { "plc-1:502", "plc-2:502" }));
+ var b = Node("node-b:4053", Driver("drv-modbus", 1, new[] { "PLC-1:502" })); // host names compare case-insensitively
+
+ FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(2);
+ }
+
+ [Fact]
+ public void Different_devices_faulted_on_each_node_are_unioned()
+ {
+ var a = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }));
+ var b = Node("node-b:4053", Driver("drv-modbus", 1, new[] { "plc-2:502" }));
+
+ FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(2);
+ }
+
+ [Fact]
+ public void The_same_host_name_under_two_drivers_is_two_devices()
+ {
+ var a = Node("node-a:4053",
+ Driver("drv-modbus", 1, new[] { "10.0.0.5" }),
+ Driver("drv-s7", 1, new[] { "10.0.0.5" }));
+
+ FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a }).ShouldBe(2);
+ }
+
+ [Fact]
+ public void A_faulted_single_host_driver_on_both_nodes_counts_once()
+ {
+ // No IHostConnectivityProbe: the driver itself is the device and carries no host name.
+ var a = Node("node-a:4053", Driver("drv-galaxy", 1, Array.Empty()));
+ var b = Node("node-b:4053", Driver("drv-galaxy", 1, Array.Empty()));
+
+ FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(1);
+ }
+
+ [Fact]
+ public void Stopped_devices_are_not_counted()
+ {
+ var a = Node("node-a:4053", Driver("drv-galaxy", 0, Array.Empty(), stopped: 3));
+
+ FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a }).ShouldBe(0);
+ }
+
+ [Fact]
+ public void A_pre_786_sender_without_host_names_is_combined_by_max_not_sum()
+ {
+ var oldSender = Node("node-a:4053", Driver("drv-modbus", 3, faultedHosts: null));
+ var newSender = Node("node-b:4053", Driver("drv-modbus", 2, new[] { "plc-1:502", "plc-2:502" }));
+
+ // Unnamed 3 (max over nodes) + named 2: the old sender's devices cannot be matched by name, so the
+ // figure is the best the wire can prove, never a doubled sum of the same unnamed devices.
+ FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { oldSender, oldSender }).ShouldBe(3);
+ FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { oldSender, newSender }).ShouldBe(5);
+ }
+
+ [Fact]
+ public void An_unreachable_node_empty_snapshot_contributes_nothing()
+ {
+ var a = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }));
+ var unreachable = Node("node-b:4053");
+
+ FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, unreachable }).ShouldBe(1);
+ FleetDiagnosticsAggregation.DistinctFaultedDevices(Array.Empty()).ShouldBe(0);
+ }
+
+ [Fact]
+ public async Task Cluster_node_ids_are_the_enabled_rows_of_the_selected_nodes_cluster_self_first()
+ {
+ var options = new DbContextOptionsBuilder()
+ .UseInMemoryDatabase(Guid.NewGuid().ToString("N")).Options;
+ await using (var seed = new OtOpcUaConfigDbContext(options))
+ {
+ seed.ClusterNodes.AddRange(
+ Row("site-a-2:4053", "SITE-A"),
+ Row("site-a-1:4053", "SITE-A"),
+ Row("site-a-3:4053", "SITE-A", enabled: false),
+ Row("site-b-1:4053", "SITE-B"));
+ await seed.SaveChangesAsync(TestContext.Current.CancellationToken);
+ }
+
+ await using var db = new OtOpcUaConfigDbContext(options);
+ var (clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(
+ db, "site-a-2:4053", TestContext.Current.CancellationToken);
+ clusterId.ShouldBe("SITE-A");
+ nodeIds.ShouldBe(new[] { "site-a-2:4053", "site-a-1:4053" });
+
+ var (unknownCluster, alone) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(
+ db, "stray:4053", TestContext.Current.CancellationToken);
+ unknownCluster.ShouldBeNull();
+ alone.ShouldBe(new[] { "stray:4053" });
+ }
+
+ [Fact]
+ public async Task A_snapshot_from_a_pre_786_sender_still_deserializes_over_the_akka_wire()
+ {
+ // The snapshot crosses Akka remoting with the default (Newtonsoft JSON) serializer. Strip the #786
+ // members from a serialized snapshot to reproduce an older sender's bytes.
+ using var system = ActorSystem.Create("diag-compat");
+ try
+ {
+ var snapshot = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }, stopped: 2));
+ var serializer = system.Serialization.FindSerializerFor(snapshot);
+ var json = Encoding.UTF8.GetString(serializer.ToBinary(snapshot));
+ json.ShouldContain("StoppedDevices");
+
+ var tree = JObject.Parse(json);
+ foreach (var prop in tree.Descendants().OfType()
+ .Where(p => p.Name is "StoppedDevices" or "FaultedHostNames").ToList())
+ prop.Remove();
+ var oldJson = tree.ToString(Formatting.None);
+ oldJson.ShouldNotContain("StoppedDevices");
+ oldJson.ShouldNotContain("FaultedHostNames");
+
+ var back = (NodeDiagnosticsSnapshot)serializer.FromBinary(Encoding.UTF8.GetBytes(oldJson), typeof(NodeDiagnosticsSnapshot));
+ var driver = back.Drivers.ShouldHaveSingleItem();
+ driver.FaultedDevices.ShouldBe(1);
+ driver.StoppedDevices.ShouldBe(0);
+ driver.FaultedHostNames.ShouldBeNull();
+ }
+ finally
+ {
+ await system.Terminate();
+ }
+ }
+
+ private static ClusterNode Row(string nodeId, string clusterId, bool enabled = true) => new()
+ {
+ NodeId = nodeId, ClusterId = clusterId, Host = nodeId.Split(':')[0],
+ ApplicationUri = "urn:" + nodeId, CreatedBy = "test", Enabled = enabled,
+ };
+}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs
index 86ca424b0..f1b6294ea 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs
@@ -4,6 +4,8 @@ using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
@@ -43,6 +45,7 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
driver.State.ShouldBe("Healthy");
driver.ConnectedDevices.ShouldBe(1);
driver.FaultedDevices.ShouldBe(0);
+ driver.StoppedDevices.ShouldBe(0);
driver.LastChangeUtc.ShouldBeGreaterThan(before);
driver.LastChangeUtc.ShouldBeLessThanOrEqualTo(DateTime.UtcNow);
}
@@ -66,6 +69,46 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
driver.FaultedDevices.ShouldBe(0);
}
+ [Fact]
+ public void Faulted_counts_only_Faulted_hosts_and_Stopped_is_reported_separately()
+ {
+ // #786: a Stopped host used to be counted as faulted.
+ var now = DateTime.UtcNow;
+ var health = new ChildHealthRecord(
+ DriverState.Degraded, now, null,
+ new[]
+ {
+ new HostConnectivityStatus("plc-a:502", HostState.Running, now),
+ new HostConnectivityStatus("plc-b:502", HostState.Faulted, now),
+ new HostConnectivityStatus("plc-c:502", HostState.Stopped, now),
+ new HostConnectivityStatus("plc-d:502", HostState.Stopped, now),
+ new HostConnectivityStatus("plc-e:502", HostState.Unknown, now),
+ },
+ now);
+
+ var row = DriverHostActor.DiagnosticsFromHealth(Guid.NewGuid(), DriverId, health);
+
+ row.ConnectedDevices.ShouldBe(1);
+ row.FaultedDevices.ShouldBe(1);
+ row.StoppedDevices.ShouldBe(2);
+ row.FaultedHostNames.ShouldBe(new[] { "plc-b:502" });
+ }
+
+ [Theory]
+ [InlineData(DriverState.Faulted, 0, 1)]
+ [InlineData(DriverState.Healthy, 1, 0)]
+ [InlineData(DriverState.Degraded, 1, 0)]
+ public void A_single_host_driver_is_one_device_with_no_host_name(DriverState state, int connected, int faulted)
+ {
+ var row = DriverHostActor.DiagnosticsFromHealth(
+ Guid.NewGuid(), DriverId, new ChildHealthRecord(state, null, null, null, DateTime.UtcNow));
+
+ row.ConnectedDevices.ShouldBe(connected);
+ row.FaultedDevices.ShouldBe(faulted);
+ row.StoppedDevices.ShouldBe(0);
+ row.FaultedHostNames.ShouldNotBeNull().ShouldBeEmpty();
+ }
+
private async Task AwaitDiagnosticsAsync(IActorRef host, Func ready)
{
NodeDiagnosticsSnapshot? snapshot = null;