Closes task #116 (GA hardening backlog). Before this commit the RedundancyStatePublisher saw PeerReachability.Unknown for every peer because the tracker had no writers — every healthy peer got degraded to the Isolated-Primary band (230) even when fully reachable. Not release-blocking (safe default), but not the full non-transparent- redundancy UX either. Two-layer probe model per docs/v2/implementation/phase-6-3-redundancy-runtime.md §Stream B: - PeerHttpProbeLoop (Stream B.1) — fast-fail layer at 2 s / 1 s timeout. Hits each peer's http://{Host}:{DashboardPort}/healthz via an injected IHttpClientFactory. Writes the HTTP bit of PeerReachability while preserving the UA bit from the last UA probe so a transient HTTP blip doesn't clobber the authoritative UA reading. - PeerUaProbeLoop (Stream B.2) — authoritative layer at 10 s / 5 s timeout. Calls DiscoveryClient.GetEndpoints against opc.tcp://{Host}: {OpcUaPort} — cheap compared to a full Session.Create, no cert trust required. Short-circuits when the HTTP probe last reported the peer unhealthy (no wasted handshakes on a known-dead endpoint), clearing the stale UaHealthy bit in that case. Both inherit from BackgroundService, follow the tick/delay/catch pattern RedundancyPublisherHostedService + ResilienceStatusPublisherHostedService established, and expose TickAsync() as internal for test drive-through. New PeerProbeOptions class carries the four intervals/timeouts so operators can tune cadence per site. Registered as singleton in Program.cs; HTTP client registered by name so the OtOpcUa handler chain (Serilog enrichers, potential future OpenTelemetry instrumentation) isn't bypassed. Tests — 9 new unit tests across PeerHttpProbeLoopTests (5) and PeerUaProbeLoopTests (4). All pass. Server.Tests total 243 → 252. Full solution build clean. Docs: v2-release-readiness.md Phase 6.3 follow-ups list marks the peer-probe bullet struck-through with a close-out note. Still deferred in Phase 6.3: - OPC UA variable-node binding (task #117 — ServiceLevel + ServerUriArray) - sp_PublishGeneration lease wrap (task #118) - Client interop matrix (task #119) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
162 lines
6.5 KiB
C#
162 lines
6.5 KiB
C#
using System.Net;
|
|
using System.Net.Http;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.Server.Hosting;
|
|
using ZB.MOM.WW.OtOpcUa.Server.Redundancy;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Server.Tests;
|
|
|
|
/// <summary>
|
|
/// Unit tests for <see cref="PeerHttpProbeLoop"/>. Drives <c>TickAsync</c> synchronously
|
|
/// via a <see cref="IHttpClientFactory"/> test double so we don't race the loop's
|
|
/// <c>Task.Delay</c>.
|
|
/// </summary>
|
|
[Trait("Category", "Unit")]
|
|
public sealed class PeerHttpProbeLoopTests : IDisposable
|
|
{
|
|
private readonly OtOpcUaConfigDbContext _db;
|
|
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
|
|
|
|
public PeerHttpProbeLoopTests()
|
|
{
|
|
var opts = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
|
.UseInMemoryDatabase($"peer-http-{Guid.NewGuid():N}")
|
|
.Options;
|
|
_db = new OtOpcUaConfigDbContext(opts);
|
|
_dbFactory = new DbContextFactory(opts);
|
|
}
|
|
|
|
public void Dispose() => _db.Dispose();
|
|
|
|
[Fact]
|
|
public async Task Tick_with_no_peers_is_a_no_op()
|
|
{
|
|
var tracker = new PeerReachabilityTracker();
|
|
var coordinator = await SeedAndInitializeAsync("A", ("A", RedundancyRole.Primary, "urn:A"));
|
|
|
|
var loop = new PeerHttpProbeLoop(coordinator, tracker,
|
|
new StubHttpClientFactory(_ => new HttpResponseMessage(HttpStatusCode.OK)),
|
|
NullLogger<PeerHttpProbeLoop>.Instance);
|
|
|
|
await loop.TickAsync(CancellationToken.None);
|
|
tracker.Get("B").ShouldBe(PeerReachability.Unknown);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Tick_marks_peer_healthy_when_healthz_returns_200()
|
|
{
|
|
var coordinator = await SeedAndInitializeAsync("A",
|
|
("A", RedundancyRole.Primary, "urn:A"),
|
|
("B", RedundancyRole.Secondary, "urn:B"));
|
|
var tracker = new PeerReachabilityTracker();
|
|
var factory = new StubHttpClientFactory(req =>
|
|
{
|
|
req.RequestUri!.AbsolutePath.ShouldBe("/healthz");
|
|
return new HttpResponseMessage(HttpStatusCode.OK);
|
|
});
|
|
|
|
var loop = new PeerHttpProbeLoop(coordinator, tracker, factory, NullLogger<PeerHttpProbeLoop>.Instance);
|
|
await loop.TickAsync(CancellationToken.None);
|
|
|
|
tracker.Get("B").HttpHealthy.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Tick_marks_peer_unhealthy_when_healthz_throws()
|
|
{
|
|
var coordinator = await SeedAndInitializeAsync("A",
|
|
("A", RedundancyRole.Primary, "urn:A"),
|
|
("B", RedundancyRole.Secondary, "urn:B"));
|
|
var tracker = new PeerReachabilityTracker();
|
|
var factory = new StubHttpClientFactory(_ => throw new HttpRequestException("no route to host"));
|
|
|
|
var loop = new PeerHttpProbeLoop(coordinator, tracker, factory, NullLogger<PeerHttpProbeLoop>.Instance);
|
|
await loop.TickAsync(CancellationToken.None);
|
|
|
|
tracker.Get("B").HttpHealthy.ShouldBeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Tick_preserves_UaHealthy_bit_when_flipping_HttpHealthy()
|
|
{
|
|
var coordinator = await SeedAndInitializeAsync("A",
|
|
("A", RedundancyRole.Primary, "urn:A"),
|
|
("B", RedundancyRole.Secondary, "urn:B"));
|
|
var tracker = new PeerReachabilityTracker();
|
|
tracker.Update("B", new PeerReachability(HttpHealthy: false, UaHealthy: true));
|
|
|
|
var factory = new StubHttpClientFactory(_ => new HttpResponseMessage(HttpStatusCode.OK));
|
|
var loop = new PeerHttpProbeLoop(coordinator, tracker, factory, NullLogger<PeerHttpProbeLoop>.Instance);
|
|
await loop.TickAsync(CancellationToken.None);
|
|
|
|
var current = tracker.Get("B");
|
|
current.HttpHealthy.ShouldBeTrue();
|
|
current.UaHealthy.ShouldBeTrue("UA bit must not be clobbered by the HTTP probe");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Tick_marks_peer_unhealthy_on_non_2xx_response()
|
|
{
|
|
var coordinator = await SeedAndInitializeAsync("A",
|
|
("A", RedundancyRole.Primary, "urn:A"),
|
|
("B", RedundancyRole.Secondary, "urn:B"));
|
|
var tracker = new PeerReachabilityTracker();
|
|
var factory = new StubHttpClientFactory(_ => new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
|
|
|
|
var loop = new PeerHttpProbeLoop(coordinator, tracker, factory, NullLogger<PeerHttpProbeLoop>.Instance);
|
|
await loop.TickAsync(CancellationToken.None);
|
|
|
|
tracker.Get("B").HttpHealthy.ShouldBeFalse();
|
|
}
|
|
|
|
// ---- fixture helpers ---------------------------------------------------
|
|
|
|
private async Task<RedundancyCoordinator> SeedAndInitializeAsync(string selfNodeId, params (string id, RedundancyRole role, string appUri)[] nodes)
|
|
{
|
|
_db.ServerClusters.Add(new ServerCluster
|
|
{
|
|
ClusterId = "c1", Name = "Warsaw", Enterprise = "zb", Site = "warsaw",
|
|
RedundancyMode = nodes.Length == 1 ? RedundancyMode.None : RedundancyMode.Warm,
|
|
CreatedBy = "test",
|
|
});
|
|
foreach (var (id, role, appUri) in nodes)
|
|
{
|
|
_db.ClusterNodes.Add(new ClusterNode
|
|
{
|
|
NodeId = id, ClusterId = "c1",
|
|
RedundancyRole = role, Host = id.ToLowerInvariant(),
|
|
ApplicationUri = appUri, CreatedBy = "test",
|
|
});
|
|
}
|
|
await _db.SaveChangesAsync();
|
|
|
|
var coordinator = new RedundancyCoordinator(_dbFactory, NullLogger<RedundancyCoordinator>.Instance, selfNodeId, "c1");
|
|
await coordinator.InitializeAsync(CancellationToken.None);
|
|
return coordinator;
|
|
}
|
|
|
|
private sealed class DbContextFactory(DbContextOptions<OtOpcUaConfigDbContext> options)
|
|
: IDbContextFactory<OtOpcUaConfigDbContext>
|
|
{
|
|
public OtOpcUaConfigDbContext CreateDbContext() => new(options);
|
|
}
|
|
|
|
private sealed class StubHttpClientFactory(Func<HttpRequestMessage, HttpResponseMessage> respond) : IHttpClientFactory
|
|
{
|
|
public HttpClient CreateClient(string name) =>
|
|
new(new StubHandler(respond), disposeHandler: true) { Timeout = TimeSpan.FromSeconds(1) };
|
|
|
|
private sealed class StubHandler(Func<HttpRequestMessage, HttpResponseMessage> respond) : HttpMessageHandler
|
|
{
|
|
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
|
=> Task.FromResult(respond(request));
|
|
}
|
|
}
|
|
}
|