Merge #786 fleet diagnostics: Faulted devices counts distinct devices across the pair, Faulted only; Stopped column
Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
@@ -2,13 +2,35 @@ using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
|
||||
/// <summary>One driver child's row in a <see cref="NodeDiagnosticsSnapshot"/>.</summary>
|
||||
/// <param name="DriverInstanceId">The driver instance's ConfigDb row id.</param>
|
||||
/// <param name="Name">The string driver-instance id (the identity every other surface keys on).</param>
|
||||
/// <param name="State">The child's last-published health state, or "Connecting" / "Stubbed".</param>
|
||||
/// <param name="ConnectedDevices">Hosts in <c>Running</c> (or 1 for a Healthy/Degraded single-host driver).</param>
|
||||
/// <param name="FaultedDevices">
|
||||
/// Hosts in <c>Faulted</c> only (or 1 for a Faulted single-host driver). Since #786 a <c>Stopped</c> host is
|
||||
/// NOT counted here — it is <paramref name="StoppedDevices"/>. A sender built before #786 still counts
|
||||
/// Stopped hosts in this figure.
|
||||
/// </param>
|
||||
/// <param name="LastChangeUtc">When the child last published a changed health snapshot (its spawn time until the first).</param>
|
||||
/// <param name="StoppedDevices">
|
||||
/// Hosts in <c>Stopped</c> (#786). Additive with a default so a snapshot from an older sender — which has no
|
||||
/// such member on the wire — still deserializes, reading 0.
|
||||
/// </param>
|
||||
/// <param name="FaultedHostNames">
|
||||
/// The <c>HostName</c>s of the hosts counted in <paramref name="FaultedDevices"/> (#786), so the AdminUI can
|
||||
/// count a device faulted on BOTH nodes of a redundant pair once. Empty for a single-host driver (the driver
|
||||
/// itself is the device); <see langword="null"/> from a sender built before #786.
|
||||
/// </param>
|
||||
public sealed record DriverInstanceDiagnostics(
|
||||
Guid DriverInstanceId,
|
||||
string Name,
|
||||
string State,
|
||||
int ConnectedDevices,
|
||||
int FaultedDevices,
|
||||
DateTime LastChangeUtc);
|
||||
DateTime LastChangeUtc,
|
||||
int StoppedDevices = 0,
|
||||
IReadOnlyList<string>? FaultedHostNames = null);
|
||||
|
||||
/// <summary>
|
||||
/// Per-node diagnostics returned by <c>IFleetDiagnosticsClient</c>. Populated by the node's
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
|
||||
|
||||
/// <summary>
|
||||
/// Cluster-wide counting for the <c>/fleet/diagnostics</c> "Faulted devices" tile (#786). Pure, so it is
|
||||
/// unit-testable without bUnit.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Owner decision 2026-11-11:</b> the tile counts DISTINCT devices across the nodes of one application
|
||||
/// cluster (a redundant pair), and only devices in <c>Faulted</c> — <c>Stopped</c> is a separate per-node
|
||||
/// column. Both nodes of a pair run the same driver instances against the same devices, so summing the two
|
||||
/// nodes' per-driver counts reported every faulted device twice.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The device identity key is (driver-instance id, host name)</b>, both compared case-insensitively:
|
||||
/// <list type="bullet">
|
||||
/// <item>
|
||||
/// The driver-instance id is <see cref="DriverInstanceDiagnostics.Name"/> — the string id from the
|
||||
/// deployed artifact, identical on both nodes of a pair (the row Guid is identical too, but can be
|
||||
/// <see cref="Guid.Empty"/> on a node with no ConfigDb row behind its spec, so the string id is the
|
||||
/// sturdier key and the one every other surface uses).
|
||||
/// </item>
|
||||
/// <item>
|
||||
/// The host name is the driver's own <c>HostConnectivityStatus.HostName</c> (e.g. <c>host:port</c>
|
||||
/// for Modbus/S7, a Platform/Engine name for Galaxy) — driver-defined from the same authored config,
|
||||
/// so the same device yields the same name on both nodes. It is scoped by the driver id because two
|
||||
/// different drivers can legitimately name the same endpoint and are counted as two devices.
|
||||
/// </item>
|
||||
/// </list>
|
||||
/// A single-host driver (no <c>IHostConnectivityProbe</c>) reports no host names: the driver instance itself
|
||||
/// is the device. Its faulted figure (0 or 1) — and any count a pre-#786 sender reports without names —
|
||||
/// is "unnamed", and unnamed counts for one driver are combined by MAX across nodes, never summed: the
|
||||
/// same unnamed device on both nodes counts once, which is the most the snapshot can prove distinct.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A node that did not answer (the client degrades to an empty snapshot) contributes nothing, so the figure
|
||||
/// is a lower bound over the nodes that did — the page shows how many answered.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class FleetDiagnosticsAggregation
|
||||
{
|
||||
/// <summary>Counts the distinct faulted devices across a set of node snapshots (normally one cluster's nodes).</summary>
|
||||
/// <param name="snapshots">One snapshot per node; duplicates of a node are harmless (the count is a set union).</param>
|
||||
/// <returns>The number of distinct (driver id, host name) devices faulted on at least one node.</returns>
|
||||
public static int DistinctFaultedDevices(IEnumerable<NodeDiagnosticsSnapshot> snapshots)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshots);
|
||||
|
||||
var named = new HashSet<(string Driver, string Host)>(DeviceKeyComparer.Instance);
|
||||
var unnamedMax = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var snapshot in snapshots)
|
||||
{
|
||||
foreach (var driver in snapshot.Drivers)
|
||||
{
|
||||
var names = driver.FaultedHostNames ?? Array.Empty<string>();
|
||||
var namedHere = 0;
|
||||
foreach (var host in names.Where(n => !string.IsNullOrWhiteSpace(n)).Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
named.Add((driver.Name, host));
|
||||
namedHere++;
|
||||
}
|
||||
|
||||
var unnamed = Math.Max(0, driver.FaultedDevices - namedHere);
|
||||
if (unnamed > 0)
|
||||
unnamedMax[driver.Name] = Math.Max(unnamedMax.GetValueOrDefault(driver.Name), unnamed);
|
||||
}
|
||||
}
|
||||
|
||||
return named.Count + unnamedMax.Values.Sum();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The node ids (<c>host:port</c>) of every enabled node in the same application cluster as
|
||||
/// <paramref name="nodeId"/>, the node itself included. A node with no <c>ClusterNode</c> row (or an empty
|
||||
/// ConfigDb) yields just itself, so the tile degrades to a single-node count rather than failing.
|
||||
/// </summary>
|
||||
/// <param name="db">The config DB context.</param>
|
||||
/// <param name="nodeId">The selected node's id, as <c>/fleet</c> shows it.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The cluster id (null when unknown) and its node ids, the selected node first.</returns>
|
||||
public static async Task<(string? ClusterId, IReadOnlyList<string> NodeIds)> LoadClusterNodeIdsAsync(
|
||||
OtOpcUaConfigDbContext db, string nodeId, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(db);
|
||||
var self = await db.ClusterNodes.AsNoTracking()
|
||||
.Where(n => n.NodeId == nodeId)
|
||||
.Select(n => n.ClusterId)
|
||||
.FirstOrDefaultAsync(ct);
|
||||
if (self is null) return (null, new[] { nodeId });
|
||||
|
||||
var peers = await db.ClusterNodes.AsNoTracking()
|
||||
.Where(n => n.ClusterId == self && n.Enabled && n.NodeId != nodeId)
|
||||
.Select(n => n.NodeId)
|
||||
.ToListAsync(ct);
|
||||
peers.Sort(StringComparer.OrdinalIgnoreCase);
|
||||
return (self, new[] { nodeId }.Concat(peers).ToList());
|
||||
}
|
||||
|
||||
private sealed class DeviceKeyComparer : IEqualityComparer<(string Driver, string Host)>
|
||||
{
|
||||
public static readonly DeviceKeyComparer Instance = new();
|
||||
|
||||
public bool Equals((string Driver, string Host) x, (string Driver, string Host) y) =>
|
||||
StringComparer.OrdinalIgnoreCase.Equals(x.Driver, y.Driver)
|
||||
&& StringComparer.OrdinalIgnoreCase.Equals(x.Host, y.Host);
|
||||
|
||||
public int GetHashCode((string Driver, string Host) key) =>
|
||||
HashCode.Combine(
|
||||
StringComparer.OrdinalIgnoreCase.GetHashCode(key.Driver),
|
||||
StringComparer.OrdinalIgnoreCase.GetHashCode(key.Host));
|
||||
}
|
||||
}
|
||||
@@ -86,9 +86,17 @@ else
|
||||
<div class="agg-label">Drivers</div>
|
||||
<div class="agg-value numeric">@_snapshot.Drivers.Count</div>
|
||||
</div>
|
||||
<div class="agg-card alert">
|
||||
<div class="agg-label">Faulted devices</div>
|
||||
<div class="agg-value numeric">@_snapshot.Drivers.Sum(d => d.FaultedDevices)</div>
|
||||
@* #786: distinct devices in Faulted across every node of the selected node's cluster (a device faulted on
|
||||
both nodes of a redundant pair counts once) — Stopped is not a fault and is a per-node column below.
|
||||
See FleetDiagnosticsAggregation for the identity key. *@
|
||||
<div class="agg-card alert"
|
||||
title="Distinct devices in Faulted state across the nodes of @(_clusterId ?? "this node's cluster") that answered (@_clusterAnswered of @_clusterNodeCount). Stopped devices are not counted; see the per-driver Stopped column.">
|
||||
<div class="agg-label">Faulted devices@(_clusterId is { } cid ? $" (cluster {cid})" : "")</div>
|
||||
<div class="agg-value numeric">@_clusterFaulted</div>
|
||||
@if (_clusterNodeCount > 1)
|
||||
{
|
||||
<div class="small text-muted">@_clusterAnswered of @_clusterNodeCount nodes answered</div>
|
||||
}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -124,6 +132,7 @@ else
|
||||
<th>State</th>
|
||||
<th class="numeric">Connected devices</th>
|
||||
<th class="numeric">Faulted devices</th>
|
||||
<th class="numeric" title="Hosts the driver reports Stopped (e.g. taken off scan) — not counted as faulted">Stopped devices</th>
|
||||
<th title="When the driver child last published a changed health snapshot (its spawn time until the first)">Last health publish</th>
|
||||
</tr>
|
||||
</thead>
|
||||
@@ -144,6 +153,7 @@ else
|
||||
<td><span class="chip @StateChipClass(d.State)">@d.State</span></td>
|
||||
<td class="numeric">@d.ConnectedDevices</td>
|
||||
<td class="numeric">@d.FaultedDevices</td>
|
||||
<td class="numeric">@d.StoppedDevices</td>
|
||||
<td>@d.LastChangeUtc.ToString("u")</td>
|
||||
</tr>
|
||||
}
|
||||
@@ -165,6 +175,12 @@ else
|
||||
private NodeDiagnosticsSnapshot? _snapshot;
|
||||
private IReadOnlyDictionary<string, string> _displayNames = new Dictionary<string, string>();
|
||||
private bool _loading;
|
||||
// #786 cluster-wide faulted-device tile: the selected node's cluster, how many of its nodes answered, and the
|
||||
// distinct faulted-device count across them.
|
||||
private string? _clusterId;
|
||||
private int _clusterNodeCount;
|
||||
private int _clusterAnswered;
|
||||
private int _clusterFaulted;
|
||||
private DateTime? _fetchedUtc;
|
||||
|
||||
// Page-lifetime cancellation: a fetch in flight when the circuit goes away is abandoned.
|
||||
@@ -222,6 +238,7 @@ else
|
||||
// The client is fail-safe by contract: a node that does not answer yields an empty snapshot.
|
||||
_snapshot = await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(_selectedNode), _cts.Token);
|
||||
_fetchedUtc = DateTime.UtcNow;
|
||||
await LoadClusterFaultedAsync(_snapshot);
|
||||
_displayNames = await LoadDisplayNamesAsync(_snapshot);
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
|
||||
@@ -235,6 +252,48 @@ else
|
||||
}
|
||||
}
|
||||
|
||||
// #786: fetch the selected node's cluster peers too and count distinct faulted devices across all of them.
|
||||
// The ConfigDb lookup is enrichment: on failure the tile falls back to the selected node alone.
|
||||
private async Task LoadClusterFaultedAsync(NodeDiagnosticsSnapshot selected)
|
||||
{
|
||||
string? clusterId = null;
|
||||
IReadOnlyList<string> nodeIds = new[] { selected.NodeId.Value };
|
||||
try
|
||||
{
|
||||
await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
|
||||
(clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(db, selected.NodeId.Value, _cts.Token);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
|
||||
{
|
||||
// ConfigDb hiccup — count this node only.
|
||||
}
|
||||
|
||||
var snapshots = new List<NodeDiagnosticsSnapshot> { selected };
|
||||
var peers = nodeIds.Where(n => !string.Equals(n, selected.NodeId.Value, StringComparison.OrdinalIgnoreCase)).ToList();
|
||||
var peerSnapshots = await Task.WhenAll(peers.Select(FetchPeerAsync));
|
||||
snapshots.AddRange(peerSnapshots.OfType<NodeDiagnosticsSnapshot>());
|
||||
|
||||
_clusterId = clusterId;
|
||||
_clusterNodeCount = nodeIds.Count;
|
||||
// The client degrades an unreachable node to an empty snapshot; one with drivers clearly answered. An empty
|
||||
// answer is indistinguishable from "no answer", so it is counted as not answered — it contributes nothing either way.
|
||||
_clusterAnswered = 1 + peerSnapshots.Count(p => p is { Drivers.Count: > 0 });
|
||||
_clusterFaulted = FleetDiagnosticsAggregation.DistinctFaultedDevices(snapshots);
|
||||
}
|
||||
|
||||
private async Task<NodeDiagnosticsSnapshot?> FetchPeerAsync(string nodeId)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(nodeId), _cts.Token);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
|
||||
{
|
||||
// A malformed peer row (NodeId.Parse) — the peer just does not contribute.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// The authored name for a driver row, else its id (no ConfigDb row, or the name lookup failed).
|
||||
private string DisplayNameOf(DriverInstanceDiagnostics d) =>
|
||||
_displayNames.TryGetValue(d.Name, out var name) ? name : d.Name;
|
||||
|
||||
@@ -2755,9 +2755,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// One child's diagnostics row. <c>Name</c> carries the string driver-instance id — the identity every
|
||||
/// other surface (/hosts, telemetry, the artifact) keys on — and <c>DriverInstanceId</c> the DB row Guid;
|
||||
/// the wire record has no slot for the authored display name. Device counts come from the driver's host
|
||||
/// statuses when it is an <c>IHostConnectivityProbe</c> (Running = connected; Stopped/Faulted = faulted;
|
||||
/// Unknown = neither); a single-host driver counts as one device, connected while it reports Healthy or
|
||||
/// Degraded and faulted while it reports Faulted. A child that has not published yet reports
|
||||
/// statuses when it is an <c>IHostConnectivityProbe</c> (Running = connected; Faulted = faulted;
|
||||
/// Stopped = stopped, counted separately since #786; Unknown = none) and the faulted hosts' names ride
|
||||
/// along so the AdminUI can de-duplicate a device across a redundant pair; a single-host driver counts as
|
||||
/// one device, connected while it reports Healthy or Degraded and faulted while it reports Faulted. A child that has not published yet reports
|
||||
/// "Connecting" (or "Stubbed") with its spawn time as the last change.
|
||||
/// </summary>
|
||||
private static DriverInstanceDiagnostics ToDiagnostics(string driverInstanceId, ChildEntry entry, ChildHealthRecord? health)
|
||||
@@ -2768,20 +2769,40 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
if (health is null)
|
||||
return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Connecting", 0, 0, entry.SpawnedUtc);
|
||||
|
||||
int connected, faulted;
|
||||
return DiagnosticsFromHealth(entry.Spec.DriverInstanceRowId, name, health);
|
||||
}
|
||||
|
||||
/// <summary>The device counts of a child that has published health (#786: Faulted and Stopped split).</summary>
|
||||
/// <param name="rowId">The driver instance's ConfigDb row id.</param>
|
||||
/// <param name="name">The string driver-instance id.</param>
|
||||
/// <param name="health">The child's last-published health.</param>
|
||||
/// <returns>The diagnostics row.</returns>
|
||||
internal static DriverInstanceDiagnostics DiagnosticsFromHealth(Guid rowId, string name, ChildHealthRecord health)
|
||||
{
|
||||
int connected, faulted, stopped;
|
||||
IReadOnlyList<string> faultedHostNames;
|
||||
if (health.HostStatuses is { Count: > 0 } hosts)
|
||||
{
|
||||
connected = hosts.Count(h => h.State == HostState.Running);
|
||||
faulted = hosts.Count(h => h.State is HostState.Stopped or HostState.Faulted);
|
||||
// #786: Faulted only. A Stopped host (e.g. a Galaxy engine an operator took off scan) is a
|
||||
// deliberate, separately-reported state, not a fault.
|
||||
var faultedHosts = hosts.Where(h => h.State == HostState.Faulted).ToArray();
|
||||
faulted = faultedHosts.Length;
|
||||
stopped = hosts.Count(h => h.State == HostState.Stopped);
|
||||
faultedHostNames = faultedHosts.Select(h => h.HostName).ToArray();
|
||||
}
|
||||
else
|
||||
{
|
||||
connected = health.State is DriverState.Healthy or DriverState.Degraded ? 1 : 0;
|
||||
faulted = health.State == DriverState.Faulted ? 1 : 0;
|
||||
stopped = 0;
|
||||
// A single-host driver is its own device: no host name; the AdminUI keys it on the instance id.
|
||||
faultedHostNames = Array.Empty<string>();
|
||||
}
|
||||
|
||||
return new DriverInstanceDiagnostics(
|
||||
entry.Spec.DriverInstanceRowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc);
|
||||
rowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc,
|
||||
StoppedDevices: stopped, FaultedHostNames: faultedHostNames);
|
||||
}
|
||||
|
||||
private void HandleDispatchFromSteady(DispatchDeployment msg)
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
using System.Text;
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// #786: the <c>/fleet/diagnostics</c> "Faulted devices" tile counts DISTINCT faulted devices across a
|
||||
/// cluster's nodes (a device faulted on both nodes of a redundant pair counts once), Faulted only.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class FleetDiagnosticsAggregationTests
|
||||
{
|
||||
private static readonly DateTime Now = DateTime.UtcNow;
|
||||
|
||||
private static DriverInstanceDiagnostics Driver(
|
||||
string name, int faulted, IReadOnlyList<string>? faultedHosts, int stopped = 0) =>
|
||||
new(Guid.NewGuid(), name, "Degraded", 0, faulted, Now, stopped, faultedHosts);
|
||||
|
||||
private static NodeDiagnosticsSnapshot Node(string nodeId, params DriverInstanceDiagnostics[] drivers) =>
|
||||
new(NodeId.Parse(nodeId), null, drivers, Now);
|
||||
|
||||
[Fact]
|
||||
public void The_same_device_faulted_on_both_nodes_of_a_pair_counts_once()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-modbus", 2, new[] { "plc-1:502", "plc-2:502" }));
|
||||
var b = Node("node-b:4053", Driver("drv-modbus", 1, new[] { "PLC-1:502" })); // host names compare case-insensitively
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Different_devices_faulted_on_each_node_are_unioned()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }));
|
||||
var b = Node("node-b:4053", Driver("drv-modbus", 1, new[] { "plc-2:502" }));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_same_host_name_under_two_drivers_is_two_devices()
|
||||
{
|
||||
var a = Node("node-a:4053",
|
||||
Driver("drv-modbus", 1, new[] { "10.0.0.5" }),
|
||||
Driver("drv-s7", 1, new[] { "10.0.0.5" }));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a }).ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_faulted_single_host_driver_on_both_nodes_counts_once()
|
||||
{
|
||||
// No IHostConnectivityProbe: the driver itself is the device and carries no host name.
|
||||
var a = Node("node-a:4053", Driver("drv-galaxy", 1, Array.Empty<string>()));
|
||||
var b = Node("node-b:4053", Driver("drv-galaxy", 1, Array.Empty<string>()));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stopped_devices_are_not_counted()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-galaxy", 0, Array.Empty<string>(), stopped: 3));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a }).ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_pre_786_sender_without_host_names_is_combined_by_max_not_sum()
|
||||
{
|
||||
var oldSender = Node("node-a:4053", Driver("drv-modbus", 3, faultedHosts: null));
|
||||
var newSender = Node("node-b:4053", Driver("drv-modbus", 2, new[] { "plc-1:502", "plc-2:502" }));
|
||||
|
||||
// Unnamed 3 (max over nodes) + named 2: the old sender's devices cannot be matched by name, so the
|
||||
// figure is the best the wire can prove, never a doubled sum of the same unnamed devices.
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { oldSender, oldSender }).ShouldBe(3);
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { oldSender, newSender }).ShouldBe(5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_unreachable_node_empty_snapshot_contributes_nothing()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }));
|
||||
var unreachable = Node("node-b:4053");
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, unreachable }).ShouldBe(1);
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(Array.Empty<NodeDiagnosticsSnapshot>()).ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cluster_node_ids_are_the_enabled_rows_of_the_selected_nodes_cluster_self_first()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
||||
.UseInMemoryDatabase(Guid.NewGuid().ToString("N")).Options;
|
||||
await using (var seed = new OtOpcUaConfigDbContext(options))
|
||||
{
|
||||
seed.ClusterNodes.AddRange(
|
||||
Row("site-a-2:4053", "SITE-A"),
|
||||
Row("site-a-1:4053", "SITE-A"),
|
||||
Row("site-a-3:4053", "SITE-A", enabled: false),
|
||||
Row("site-b-1:4053", "SITE-B"));
|
||||
await seed.SaveChangesAsync(TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
await using var db = new OtOpcUaConfigDbContext(options);
|
||||
var (clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(
|
||||
db, "site-a-2:4053", TestContext.Current.CancellationToken);
|
||||
clusterId.ShouldBe("SITE-A");
|
||||
nodeIds.ShouldBe(new[] { "site-a-2:4053", "site-a-1:4053" });
|
||||
|
||||
var (unknownCluster, alone) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(
|
||||
db, "stray:4053", TestContext.Current.CancellationToken);
|
||||
unknownCluster.ShouldBeNull();
|
||||
alone.ShouldBe(new[] { "stray:4053" });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_snapshot_from_a_pre_786_sender_still_deserializes_over_the_akka_wire()
|
||||
{
|
||||
// The snapshot crosses Akka remoting with the default (Newtonsoft JSON) serializer. Strip the #786
|
||||
// members from a serialized snapshot to reproduce an older sender's bytes.
|
||||
using var system = ActorSystem.Create("diag-compat");
|
||||
try
|
||||
{
|
||||
var snapshot = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }, stopped: 2));
|
||||
var serializer = system.Serialization.FindSerializerFor(snapshot);
|
||||
var json = Encoding.UTF8.GetString(serializer.ToBinary(snapshot));
|
||||
json.ShouldContain("StoppedDevices");
|
||||
|
||||
var tree = JObject.Parse(json);
|
||||
foreach (var prop in tree.Descendants().OfType<JProperty>()
|
||||
.Where(p => p.Name is "StoppedDevices" or "FaultedHostNames").ToList())
|
||||
prop.Remove();
|
||||
var oldJson = tree.ToString(Formatting.None);
|
||||
oldJson.ShouldNotContain("StoppedDevices");
|
||||
oldJson.ShouldNotContain("FaultedHostNames");
|
||||
|
||||
var back = (NodeDiagnosticsSnapshot)serializer.FromBinary(Encoding.UTF8.GetBytes(oldJson), typeof(NodeDiagnosticsSnapshot));
|
||||
var driver = back.Drivers.ShouldHaveSingleItem();
|
||||
driver.FaultedDevices.ShouldBe(1);
|
||||
driver.StoppedDevices.ShouldBe(0);
|
||||
driver.FaultedHostNames.ShouldBeNull();
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
private static ClusterNode Row(string nodeId, string clusterId, bool enabled = true) => new()
|
||||
{
|
||||
NodeId = nodeId, ClusterId = clusterId, Host = nodeId.Split(':')[0],
|
||||
ApplicationUri = "urn:" + nodeId, CreatedBy = "test", Enabled = enabled,
|
||||
};
|
||||
}
|
||||
+43
@@ -4,6 +4,8 @@ using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
@@ -43,6 +45,7 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
|
||||
driver.State.ShouldBe("Healthy");
|
||||
driver.ConnectedDevices.ShouldBe(1);
|
||||
driver.FaultedDevices.ShouldBe(0);
|
||||
driver.StoppedDevices.ShouldBe(0);
|
||||
driver.LastChangeUtc.ShouldBeGreaterThan(before);
|
||||
driver.LastChangeUtc.ShouldBeLessThanOrEqualTo(DateTime.UtcNow);
|
||||
}
|
||||
@@ -66,6 +69,46 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
|
||||
driver.FaultedDevices.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Faulted_counts_only_Faulted_hosts_and_Stopped_is_reported_separately()
|
||||
{
|
||||
// #786: a Stopped host used to be counted as faulted.
|
||||
var now = DateTime.UtcNow;
|
||||
var health = new ChildHealthRecord(
|
||||
DriverState.Degraded, now, null,
|
||||
new[]
|
||||
{
|
||||
new HostConnectivityStatus("plc-a:502", HostState.Running, now),
|
||||
new HostConnectivityStatus("plc-b:502", HostState.Faulted, now),
|
||||
new HostConnectivityStatus("plc-c:502", HostState.Stopped, now),
|
||||
new HostConnectivityStatus("plc-d:502", HostState.Stopped, now),
|
||||
new HostConnectivityStatus("plc-e:502", HostState.Unknown, now),
|
||||
},
|
||||
now);
|
||||
|
||||
var row = DriverHostActor.DiagnosticsFromHealth(Guid.NewGuid(), DriverId, health);
|
||||
|
||||
row.ConnectedDevices.ShouldBe(1);
|
||||
row.FaultedDevices.ShouldBe(1);
|
||||
row.StoppedDevices.ShouldBe(2);
|
||||
row.FaultedHostNames.ShouldBe(new[] { "plc-b:502" });
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DriverState.Faulted, 0, 1)]
|
||||
[InlineData(DriverState.Healthy, 1, 0)]
|
||||
[InlineData(DriverState.Degraded, 1, 0)]
|
||||
public void A_single_host_driver_is_one_device_with_no_host_name(DriverState state, int connected, int faulted)
|
||||
{
|
||||
var row = DriverHostActor.DiagnosticsFromHealth(
|
||||
Guid.NewGuid(), DriverId, new ChildHealthRecord(state, null, null, null, DateTime.UtcNow));
|
||||
|
||||
row.ConnectedDevices.ShouldBe(connected);
|
||||
row.FaultedDevices.ShouldBe(faulted);
|
||||
row.StoppedDevices.ShouldBe(0);
|
||||
row.FaultedHostNames.ShouldNotBeNull().ShouldBeEmpty();
|
||||
}
|
||||
|
||||
private async Task<NodeDiagnosticsSnapshot> AwaitDiagnosticsAsync(IActorRef host, Func<NodeDiagnosticsSnapshot, bool> ready)
|
||||
{
|
||||
NodeDiagnosticsSnapshot? snapshot = null;
|
||||
|
||||
Reference in New Issue
Block a user