fix(adminui): Faulted-devices tile counts distinct Faulted devices across the cluster pair (#786)
Owner decision 2026-11-11: /fleet/diagnostics "Faulted devices" counts DISTINCT devices across the nodes of the selected node's cluster, Faulted state only. - DriverHostActor: FaultedDevices = HostState.Faulted only (Stopped no longer counted); new StoppedDevices + FaultedHostNames. Counting extracted to the testable DiagnosticsFromHealth. - DriverInstanceDiagnostics: StoppedDevices (default 0) and FaultedHostNames (default null) are additive with defaults, so a pre-#786 sender's snapshot still deserializes over Akka's default JSON serializer (pinned by a test). - FleetDiagnosticsAggregation (pure): identity key = (driver-instance string id, HostName), case-insensitive; a single-host driver / nameless pre-#786 count is combined by MAX across nodes, never summed. Cluster peers come from enabled ClusterNode rows of the selected node's ClusterId. - FleetDiagnostics.razor: tile fetches the cluster peers and shows the distinct count + "n of m nodes answered"; per-driver table gains a Stopped devices column. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -0,0 +1,165 @@
|
||||
using System.Text;
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// #786: the <c>/fleet/diagnostics</c> "Faulted devices" tile counts DISTINCT faulted devices across a
|
||||
/// cluster's nodes (a device faulted on both nodes of a redundant pair counts once), Faulted only.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class FleetDiagnosticsAggregationTests
|
||||
{
|
||||
private static readonly DateTime Now = DateTime.UtcNow;
|
||||
|
||||
private static DriverInstanceDiagnostics Driver(
|
||||
string name, int faulted, IReadOnlyList<string>? faultedHosts, int stopped = 0) =>
|
||||
new(Guid.NewGuid(), name, "Degraded", 0, faulted, Now, stopped, faultedHosts);
|
||||
|
||||
private static NodeDiagnosticsSnapshot Node(string nodeId, params DriverInstanceDiagnostics[] drivers) =>
|
||||
new(NodeId.Parse(nodeId), null, drivers, Now);
|
||||
|
||||
[Fact]
|
||||
public void The_same_device_faulted_on_both_nodes_of_a_pair_counts_once()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-modbus", 2, new[] { "plc-1:502", "plc-2:502" }));
|
||||
var b = Node("node-b:4053", Driver("drv-modbus", 1, new[] { "PLC-1:502" })); // host names compare case-insensitively
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Different_devices_faulted_on_each_node_are_unioned()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }));
|
||||
var b = Node("node-b:4053", Driver("drv-modbus", 1, new[] { "plc-2:502" }));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_same_host_name_under_two_drivers_is_two_devices()
|
||||
{
|
||||
var a = Node("node-a:4053",
|
||||
Driver("drv-modbus", 1, new[] { "10.0.0.5" }),
|
||||
Driver("drv-s7", 1, new[] { "10.0.0.5" }));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a }).ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_faulted_single_host_driver_on_both_nodes_counts_once()
|
||||
{
|
||||
// No IHostConnectivityProbe: the driver itself is the device and carries no host name.
|
||||
var a = Node("node-a:4053", Driver("drv-galaxy", 1, Array.Empty<string>()));
|
||||
var b = Node("node-b:4053", Driver("drv-galaxy", 1, Array.Empty<string>()));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stopped_devices_are_not_counted()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-galaxy", 0, Array.Empty<string>(), stopped: 3));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a }).ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_pre_786_sender_without_host_names_is_combined_by_max_not_sum()
|
||||
{
|
||||
var oldSender = Node("node-a:4053", Driver("drv-modbus", 3, faultedHosts: null));
|
||||
var newSender = Node("node-b:4053", Driver("drv-modbus", 2, new[] { "plc-1:502", "plc-2:502" }));
|
||||
|
||||
// Unnamed 3 (max over nodes) + named 2: the old sender's devices cannot be matched by name, so the
|
||||
// figure is the best the wire can prove, never a doubled sum of the same unnamed devices.
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { oldSender, oldSender }).ShouldBe(3);
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { oldSender, newSender }).ShouldBe(5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_unreachable_node_empty_snapshot_contributes_nothing()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }));
|
||||
var unreachable = Node("node-b:4053");
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, unreachable }).ShouldBe(1);
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(Array.Empty<NodeDiagnosticsSnapshot>()).ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cluster_node_ids_are_the_enabled_rows_of_the_selected_nodes_cluster_self_first()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
||||
.UseInMemoryDatabase(Guid.NewGuid().ToString("N")).Options;
|
||||
await using (var seed = new OtOpcUaConfigDbContext(options))
|
||||
{
|
||||
seed.ClusterNodes.AddRange(
|
||||
Row("site-a-2:4053", "SITE-A"),
|
||||
Row("site-a-1:4053", "SITE-A"),
|
||||
Row("site-a-3:4053", "SITE-A", enabled: false),
|
||||
Row("site-b-1:4053", "SITE-B"));
|
||||
await seed.SaveChangesAsync(TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
await using var db = new OtOpcUaConfigDbContext(options);
|
||||
var (clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(
|
||||
db, "site-a-2:4053", TestContext.Current.CancellationToken);
|
||||
clusterId.ShouldBe("SITE-A");
|
||||
nodeIds.ShouldBe(new[] { "site-a-2:4053", "site-a-1:4053" });
|
||||
|
||||
var (unknownCluster, alone) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(
|
||||
db, "stray:4053", TestContext.Current.CancellationToken);
|
||||
unknownCluster.ShouldBeNull();
|
||||
alone.ShouldBe(new[] { "stray:4053" });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_snapshot_from_a_pre_786_sender_still_deserializes_over_the_akka_wire()
|
||||
{
|
||||
// The snapshot crosses Akka remoting with the default (Newtonsoft JSON) serializer. Strip the #786
|
||||
// members from a serialized snapshot to reproduce an older sender's bytes.
|
||||
using var system = ActorSystem.Create("diag-compat");
|
||||
try
|
||||
{
|
||||
var snapshot = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }, stopped: 2));
|
||||
var serializer = system.Serialization.FindSerializerFor(snapshot);
|
||||
var json = Encoding.UTF8.GetString(serializer.ToBinary(snapshot));
|
||||
json.ShouldContain("StoppedDevices");
|
||||
|
||||
var tree = JObject.Parse(json);
|
||||
foreach (var prop in tree.Descendants().OfType<JProperty>()
|
||||
.Where(p => p.Name is "StoppedDevices" or "FaultedHostNames").ToList())
|
||||
prop.Remove();
|
||||
var oldJson = tree.ToString(Formatting.None);
|
||||
oldJson.ShouldNotContain("StoppedDevices");
|
||||
oldJson.ShouldNotContain("FaultedHostNames");
|
||||
|
||||
var back = (NodeDiagnosticsSnapshot)serializer.FromBinary(Encoding.UTF8.GetBytes(oldJson), typeof(NodeDiagnosticsSnapshot));
|
||||
var driver = back.Drivers.ShouldHaveSingleItem();
|
||||
driver.FaultedDevices.ShouldBe(1);
|
||||
driver.StoppedDevices.ShouldBe(0);
|
||||
driver.FaultedHostNames.ShouldBeNull();
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
private static ClusterNode Row(string nodeId, string clusterId, bool enabled = true) => new()
|
||||
{
|
||||
NodeId = nodeId, ClusterId = clusterId, Host = nodeId.Split(':')[0],
|
||||
ApplicationUri = "urn:" + nodeId, CreatedBy = "test", Enabled = enabled,
|
||||
};
|
||||
}
|
||||
+43
@@ -4,6 +4,8 @@ using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
@@ -43,6 +45,7 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
|
||||
driver.State.ShouldBe("Healthy");
|
||||
driver.ConnectedDevices.ShouldBe(1);
|
||||
driver.FaultedDevices.ShouldBe(0);
|
||||
driver.StoppedDevices.ShouldBe(0);
|
||||
driver.LastChangeUtc.ShouldBeGreaterThan(before);
|
||||
driver.LastChangeUtc.ShouldBeLessThanOrEqualTo(DateTime.UtcNow);
|
||||
}
|
||||
@@ -66,6 +69,46 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
|
||||
driver.FaultedDevices.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Faulted_counts_only_Faulted_hosts_and_Stopped_is_reported_separately()
|
||||
{
|
||||
// #786: a Stopped host used to be counted as faulted.
|
||||
var now = DateTime.UtcNow;
|
||||
var health = new ChildHealthRecord(
|
||||
DriverState.Degraded, now, null,
|
||||
new[]
|
||||
{
|
||||
new HostConnectivityStatus("plc-a:502", HostState.Running, now),
|
||||
new HostConnectivityStatus("plc-b:502", HostState.Faulted, now),
|
||||
new HostConnectivityStatus("plc-c:502", HostState.Stopped, now),
|
||||
new HostConnectivityStatus("plc-d:502", HostState.Stopped, now),
|
||||
new HostConnectivityStatus("plc-e:502", HostState.Unknown, now),
|
||||
},
|
||||
now);
|
||||
|
||||
var row = DriverHostActor.DiagnosticsFromHealth(Guid.NewGuid(), DriverId, health);
|
||||
|
||||
row.ConnectedDevices.ShouldBe(1);
|
||||
row.FaultedDevices.ShouldBe(1);
|
||||
row.StoppedDevices.ShouldBe(2);
|
||||
row.FaultedHostNames.ShouldBe(new[] { "plc-b:502" });
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DriverState.Faulted, 0, 1)]
|
||||
[InlineData(DriverState.Healthy, 1, 0)]
|
||||
[InlineData(DriverState.Degraded, 1, 0)]
|
||||
public void A_single_host_driver_is_one_device_with_no_host_name(DriverState state, int connected, int faulted)
|
||||
{
|
||||
var row = DriverHostActor.DiagnosticsFromHealth(
|
||||
Guid.NewGuid(), DriverId, new ChildHealthRecord(state, null, null, null, DateTime.UtcNow));
|
||||
|
||||
row.ConnectedDevices.ShouldBe(connected);
|
||||
row.FaultedDevices.ShouldBe(faulted);
|
||||
row.StoppedDevices.ShouldBe(0);
|
||||
row.FaultedHostNames.ShouldNotBeNull().ShouldBeEmpty();
|
||||
}
|
||||
|
||||
private async Task<NodeDiagnosticsSnapshot> AwaitDiagnosticsAsync(IActorRef host, Func<NodeDiagnosticsSnapshot, bool> ready)
|
||||
{
|
||||
NodeDiagnosticsSnapshot? snapshot = null;
|
||||
|
||||
Reference in New Issue
Block a user