Three fixes for the SubscriptionManager/MxAccessClient subscription pipeline: 1. Serialize Subscribe and UnsubscribeClient with a SemaphoreSlim gate to prevent race where old-session unsubscribe removes new-session COM subscriptions. CreateMxAccessSubscriptionsAsync is now awaited instead of fire-and-forget. 2. Fix dual VTQ delivery in MxAccessClient.OnDataChange — each update was delivered twice (once via stored callback, once via OnTagValueChanged property). Now uses stored callback as the single delivery path. 3. Store pending tag addresses when CreateMxAccessSubscriptionsAsync fails (MxAccess down) and retry them on reconnect via NotifyReconnection/RetryPendingSubscriptionsAsync. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
245 lines
11 KiB
C#
245 lines
11 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using FluentAssertions;
|
|
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
|
using Xunit;
|
|
using ZB.MOM.WW.LmxProxy.Host.Domain;
|
|
using ZB.MOM.WW.LmxProxy.Host.Health;
|
|
using HealthCheckService = ZB.MOM.WW.LmxProxy.Host.Health.HealthCheckService;
|
|
using ZB.MOM.WW.LmxProxy.Host.Metrics;
|
|
using ZB.MOM.WW.LmxProxy.Host.Subscriptions;
|
|
|
|
namespace ZB.MOM.WW.LmxProxy.Host.Tests.Health
|
|
{
|
|
public class HealthCheckServiceTests
|
|
{
|
|
private class FakeScadaClient : IScadaClient
|
|
{
|
|
public bool IsConnected { get; set; } = true;
|
|
public ConnectionState ConnectionState { get; set; } = ConnectionState.Connected;
|
|
public event EventHandler<ConnectionStateChangedEventArgs>? ConnectionStateChanged;
|
|
public Task ConnectAsync(CancellationToken ct = default) => Task.CompletedTask;
|
|
public Task DisconnectAsync(CancellationToken ct = default) => Task.CompletedTask;
|
|
public Task<Vtq> ReadAsync(string address, CancellationToken ct = default) =>
|
|
Task.FromResult(Vtq.Good(42.0));
|
|
public Task<IReadOnlyDictionary<string, Vtq>> ReadBatchAsync(IEnumerable<string> addresses, CancellationToken ct = default) =>
|
|
Task.FromResult<IReadOnlyDictionary<string, Vtq>>(new Dictionary<string, Vtq>());
|
|
public Task WriteAsync(string address, object value, CancellationToken ct = default) => Task.CompletedTask;
|
|
public Task WriteBatchAsync(IReadOnlyDictionary<string, object> values, CancellationToken ct = default) => Task.CompletedTask;
|
|
public Task<(bool flagReached, int elapsedMs)> WriteBatchAndWaitAsync(
|
|
IReadOnlyDictionary<string, object> values, string flagTag, object flagValue,
|
|
int timeoutMs, int pollIntervalMs, CancellationToken ct = default) =>
|
|
Task.FromResult((false, 0));
|
|
public Task<ProbeResult> ProbeConnectionAsync(string testTagAddress, int timeoutMs, CancellationToken ct = default) =>
|
|
Task.FromResult(ProbeResult.Healthy(Quality.Good, DateTime.UtcNow));
|
|
public Task UnsubscribeByAddressAsync(IEnumerable<string> addresses) => Task.CompletedTask;
|
|
public Task<IAsyncDisposable> SubscribeAsync(IEnumerable<string> addresses, Action<string, Vtq> callback, CancellationToken ct = default) =>
|
|
Task.FromResult<IAsyncDisposable>(new FakeHandle());
|
|
public ValueTask DisposeAsync() => default;
|
|
internal void FireEvent() => ConnectionStateChanged?.Invoke(this, null!);
|
|
private class FakeHandle : IAsyncDisposable { public ValueTask DisposeAsync() => default; }
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReturnsHealthy_WhenConnectedAndNormalMetrics()
|
|
{
|
|
var client = new FakeScadaClient { IsConnected = true, ConnectionState = ConnectionState.Connected };
|
|
using var sm = new SubscriptionManager(client);
|
|
using var pm = new PerformanceMetrics();
|
|
pm.RecordOperation("Read", TimeSpan.FromMilliseconds(10), true);
|
|
|
|
var svc = new HealthCheckService(client, sm, pm);
|
|
|
|
var result = await svc.CheckHealthAsync(new HealthCheckContext());
|
|
|
|
result.Status.Should().Be(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReturnsUnhealthy_WhenNotConnected()
|
|
{
|
|
var client = new FakeScadaClient { IsConnected = false, ConnectionState = ConnectionState.Disconnected };
|
|
using var sm = new SubscriptionManager(client);
|
|
using var pm = new PerformanceMetrics();
|
|
|
|
var svc = new HealthCheckService(client, sm, pm);
|
|
|
|
var result = await svc.CheckHealthAsync(new HealthCheckContext());
|
|
|
|
result.Status.Should().Be(HealthStatus.Unhealthy);
|
|
result.Description.Should().Contain("not connected");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReturnsDegraded_WhenSuccessRateBelow50Percent()
|
|
{
|
|
var client = new FakeScadaClient { IsConnected = true };
|
|
using var sm = new SubscriptionManager(client);
|
|
using var pm = new PerformanceMetrics();
|
|
|
|
// Record 200 operations with 40% success rate
|
|
for (int i = 0; i < 80; i++)
|
|
pm.RecordOperation("Read", TimeSpan.FromMilliseconds(10), true);
|
|
for (int i = 0; i < 120; i++)
|
|
pm.RecordOperation("Read", TimeSpan.FromMilliseconds(10), false);
|
|
|
|
var svc = new HealthCheckService(client, sm, pm);
|
|
|
|
var result = await svc.CheckHealthAsync(new HealthCheckContext());
|
|
|
|
result.Status.Should().Be(HealthStatus.Degraded);
|
|
result.Description.Should().Contain("success rate");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReturnsDegraded_WhenClientCountOver100()
|
|
{
|
|
var client = new FakeScadaClient { IsConnected = true };
|
|
using var sm = new SubscriptionManager(client);
|
|
using var pm = new PerformanceMetrics();
|
|
|
|
// Create 101 subscriptions to exceed the threshold
|
|
for (int i = 0; i < 101; i++)
|
|
{
|
|
using var cts = new CancellationTokenSource();
|
|
await sm.SubscribeAsync("client-" + i, new[] { "tag1" }, cts.Token);
|
|
}
|
|
|
|
var svc = new HealthCheckService(client, sm, pm);
|
|
|
|
var result = await svc.CheckHealthAsync(new HealthCheckContext());
|
|
|
|
result.Status.Should().Be(HealthStatus.Degraded);
|
|
result.Description.Should().Contain("client count");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DoesNotFlagLowSuccessRate_Under100Operations()
|
|
{
|
|
var client = new FakeScadaClient { IsConnected = true };
|
|
using var sm = new SubscriptionManager(client);
|
|
using var pm = new PerformanceMetrics();
|
|
|
|
// Record 50 operations with 0% success rate (under 100 threshold)
|
|
for (int i = 0; i < 50; i++)
|
|
pm.RecordOperation("Read", TimeSpan.FromMilliseconds(10), false);
|
|
|
|
var svc = new HealthCheckService(client, sm, pm);
|
|
|
|
var result = await svc.CheckHealthAsync(new HealthCheckContext());
|
|
|
|
result.Status.Should().Be(HealthStatus.Healthy);
|
|
}
|
|
}
|
|
|
|
public class DetailedHealthCheckServiceTests
|
|
{
|
|
private class FakeScadaClient : IScadaClient
|
|
{
|
|
public bool IsConnected { get; set; } = true;
|
|
public ConnectionState ConnectionState { get; set; } = ConnectionState.Connected;
|
|
public Vtq? ReadResult { get; set; }
|
|
public Exception? ReadException { get; set; }
|
|
public event EventHandler<ConnectionStateChangedEventArgs>? ConnectionStateChanged;
|
|
public Task ConnectAsync(CancellationToken ct = default) => Task.CompletedTask;
|
|
public Task DisconnectAsync(CancellationToken ct = default) => Task.CompletedTask;
|
|
public Task<Vtq> ReadAsync(string address, CancellationToken ct = default)
|
|
{
|
|
if (ReadException != null) throw ReadException;
|
|
return Task.FromResult(ReadResult ?? Vtq.Good(true));
|
|
}
|
|
public Task<IReadOnlyDictionary<string, Vtq>> ReadBatchAsync(IEnumerable<string> addresses, CancellationToken ct = default) =>
|
|
Task.FromResult<IReadOnlyDictionary<string, Vtq>>(new Dictionary<string, Vtq>());
|
|
public Task WriteAsync(string address, object value, CancellationToken ct = default) => Task.CompletedTask;
|
|
public Task WriteBatchAsync(IReadOnlyDictionary<string, object> values, CancellationToken ct = default) => Task.CompletedTask;
|
|
public Task<(bool flagReached, int elapsedMs)> WriteBatchAndWaitAsync(
|
|
IReadOnlyDictionary<string, object> values, string flagTag, object flagValue,
|
|
int timeoutMs, int pollIntervalMs, CancellationToken ct = default) =>
|
|
Task.FromResult((false, 0));
|
|
public Task<ProbeResult> ProbeConnectionAsync(string testTagAddress, int timeoutMs, CancellationToken ct = default) =>
|
|
Task.FromResult(ProbeResult.Healthy(Quality.Good, DateTime.UtcNow));
|
|
public Task UnsubscribeByAddressAsync(IEnumerable<string> addresses) => Task.CompletedTask;
|
|
public Task<IAsyncDisposable> SubscribeAsync(IEnumerable<string> addresses, Action<string, Vtq> callback, CancellationToken ct = default) =>
|
|
Task.FromResult<IAsyncDisposable>(new FakeHandle());
|
|
public ValueTask DisposeAsync() => default;
|
|
internal void FireEvent() => ConnectionStateChanged?.Invoke(this, null!);
|
|
private class FakeHandle : IAsyncDisposable { public ValueTask DisposeAsync() => default; }
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReturnsUnhealthy_WhenNotConnected()
|
|
{
|
|
var client = new FakeScadaClient { IsConnected = false };
|
|
var svc = new DetailedHealthCheckService(client);
|
|
|
|
var result = await svc.CheckHealthAsync(new HealthCheckContext());
|
|
|
|
result.Status.Should().Be(HealthStatus.Unhealthy);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReturnsHealthy_WhenTestTagGoodAndRecent()
|
|
{
|
|
var client = new FakeScadaClient
|
|
{
|
|
IsConnected = true,
|
|
ReadResult = Vtq.New(true, DateTime.UtcNow, Quality.Good)
|
|
};
|
|
var svc = new DetailedHealthCheckService(client);
|
|
|
|
var result = await svc.CheckHealthAsync(new HealthCheckContext());
|
|
|
|
result.Status.Should().Be(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReturnsDegraded_WhenTestTagQualityNotGood()
|
|
{
|
|
var client = new FakeScadaClient
|
|
{
|
|
IsConnected = true,
|
|
ReadResult = Vtq.New(true, DateTime.UtcNow, Quality.Uncertain)
|
|
};
|
|
var svc = new DetailedHealthCheckService(client);
|
|
|
|
var result = await svc.CheckHealthAsync(new HealthCheckContext());
|
|
|
|
result.Status.Should().Be(HealthStatus.Degraded);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReturnsDegraded_WhenTestTagTimestampStale()
|
|
{
|
|
var client = new FakeScadaClient
|
|
{
|
|
IsConnected = true,
|
|
ReadResult = Vtq.New(true, DateTime.UtcNow.AddMinutes(-10), Quality.Good)
|
|
};
|
|
var svc = new DetailedHealthCheckService(client);
|
|
|
|
var result = await svc.CheckHealthAsync(new HealthCheckContext());
|
|
|
|
result.Status.Should().Be(HealthStatus.Degraded);
|
|
result.Description.Should().Contain("stale");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReturnsDegraded_WhenTestTagReadThrows()
|
|
{
|
|
var client = new FakeScadaClient
|
|
{
|
|
IsConnected = true,
|
|
ReadException = new InvalidOperationException("COM error")
|
|
};
|
|
var svc = new DetailedHealthCheckService(client);
|
|
|
|
var result = await svc.CheckHealthAsync(new HealthCheckContext());
|
|
|
|
result.Status.Should().Be(HealthStatus.Degraded);
|
|
result.Description.Should().Contain("Could not read test tag");
|
|
}
|
|
}
|
|
}
|