344bdc4da5
The dashboard derived every leader chip from AkkaClusterHealthCheck's `leader` data — ClusterState.Leader, the LOWEST-ADDRESSED Up member. That is not the node in charge. The node holding the singletons, gated for writes, and answering 200 on /health/active is the OLDEST Up member. The two agree only until some node restarts, after which the restarted node rejoins youngest but keeps its address. Observed live on the rebuilt OtOpcUa rig during the Health 0.3.0 work: leader central-1, active node central-2. The dashboard would have put the Active chip on central-1 while /health/active said central-2 — the chip contradicting the tier it exists to visualise. The Phase 4 acceptance run missed it because the rig had just been formed, the one state in which the two orderings agree. Health 0.3.0's active tier now publishes `activeNode` on every member, including standbys, so the fix is to prefer it and keep `leader` as the fallback for an instance on an older Health or whose active tier was not probed. The instance card now shows `active` and `leader` side by side rather than swapping one for the other: they are legitimately different nodes, and showing both turns a silent contradiction into something readable at 3am. Tests: 177 (+3) — precedence when the two disagree, a standby naming the active node rather than itself, and the fallback still working with no activeNode.
198 lines
6.6 KiB
C#
198 lines
6.6 KiB
C#
using System.Diagnostics.Metrics;
|
|
using System.Net;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using ZB.MOM.WW.Overview.Polling;
|
|
|
|
namespace ZB.MOM.WW.Overview.Tests;
|
|
|
|
/// <summary>A meter factory with no listener attached, for tests that only need the poller to run.</summary>
|
|
internal sealed class DummyMeterFactory : IMeterFactory
|
|
{
|
|
private readonly List<Meter> _meters = [];
|
|
|
|
public Meter Create(MeterOptions options)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(options);
|
|
|
|
// Stamp ourselves as the scope, exactly as the real factory does. Meter names are global,
|
|
// so a MeterListener that matched on name alone would pick up meters created by tests
|
|
// running in parallel; the scope is what makes one test's instruments identifiable.
|
|
options.Scope = this;
|
|
|
|
var meter = new Meter(options);
|
|
_meters.Add(meter);
|
|
return meter;
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
foreach (var meter in _meters)
|
|
meter.Dispose();
|
|
|
|
_meters.Clear();
|
|
}
|
|
}
|
|
|
|
/// <summary>A clock the test drives by hand.</summary>
|
|
/// <remarks>
|
|
/// Only <see cref="GetUtcNow"/> is overridden: the poller's sweep — the part with rules worth
|
|
/// pinning — is exercised directly, so no test here needs a fake timer.
|
|
/// </remarks>
|
|
internal sealed class FakeTimeProvider(DateTimeOffset start) : TimeProvider
|
|
{
|
|
private DateTimeOffset _now = start;
|
|
|
|
public override DateTimeOffset GetUtcNow() => _now;
|
|
|
|
public void Advance(TimeSpan by) => _now += by;
|
|
}
|
|
|
|
/// <summary>Serves canned responses per absolute URL and counts what was requested.</summary>
|
|
internal sealed class RoutingHandler : HttpMessageHandler
|
|
{
|
|
private readonly Dictionary<string, Func<HttpResponseMessage>> _routes = new(StringComparer.OrdinalIgnoreCase);
|
|
|
|
public List<string> Requests { get; } = [];
|
|
|
|
public RoutingHandler Json(string url, HttpStatusCode status, string body)
|
|
{
|
|
_routes[url] = () => new HttpResponseMessage(status)
|
|
{
|
|
Content = new StringContent(body, Encoding.UTF8, "application/json"),
|
|
};
|
|
return this;
|
|
}
|
|
|
|
/// <summary>
|
|
/// A status-code-only response with an EMPTY body — what the active tier actually returns when
|
|
/// it is a bare gate endpoint rather than a full health report.
|
|
/// </summary>
|
|
public RoutingHandler Status(string url, HttpStatusCode status)
|
|
{
|
|
_routes[url] = () => new HttpResponseMessage(status)
|
|
{
|
|
Content = new StringContent(string.Empty, Encoding.UTF8, "application/json"),
|
|
};
|
|
return this;
|
|
}
|
|
|
|
public RoutingHandler Fails(string url, string message)
|
|
{
|
|
_routes[url] = () => throw new HttpRequestException(message);
|
|
return this;
|
|
}
|
|
|
|
/// <summary>Lets a wrapping handler delegate here without re-entering the protected member.</summary>
|
|
public Task<HttpResponseMessage> HandleAsync(HttpRequestMessage request) =>
|
|
SendAsync(request, CancellationToken.None);
|
|
|
|
protected override Task<HttpResponseMessage> SendAsync(
|
|
HttpRequestMessage request, CancellationToken cancellationToken)
|
|
{
|
|
var url = request.RequestUri!.ToString();
|
|
|
|
lock (Requests)
|
|
Requests.Add(url);
|
|
|
|
if (!_routes.TryGetValue(url, out var factory))
|
|
return Task.FromException<HttpResponseMessage>(new HttpRequestException($"no route for {url}"));
|
|
|
|
try
|
|
{
|
|
return Task.FromResult(factory());
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
return Task.FromException<HttpResponseMessage>(ex);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>Hands every named client the same test handler.</summary>
|
|
internal sealed class StubHttpClientFactory(HttpMessageHandler handler) : IHttpClientFactory
|
|
{
|
|
public HttpClient CreateClient(string name) =>
|
|
new(handler, disposeHandler: false) { Timeout = Timeout.InfiniteTimeSpan };
|
|
}
|
|
|
|
/// <summary>Builds canonical health bodies so tests state intent, not JSON.</summary>
|
|
internal static class HealthBody
|
|
{
|
|
public static string Ready(string status, params (string Name, string Status, string? Leader)[] entries)
|
|
{
|
|
var payload = new
|
|
{
|
|
status,
|
|
totalDurationMs = 1.5,
|
|
entries = entries.ToDictionary(
|
|
e => e.Name,
|
|
e => (object)new
|
|
{
|
|
status = e.Status,
|
|
description = $"{e.Name} says {e.Status}",
|
|
durationMs = 0.5,
|
|
data = e.Leader is null ? null : new Dictionary<string, object> { ["leader"] = e.Leader },
|
|
},
|
|
StringComparer.Ordinal),
|
|
};
|
|
|
|
return JsonSerializer.Serialize(payload);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Builds a body whose entries carry arbitrary <c>data</c>, for the cases where the key under
|
|
/// test is not <c>leader</c> — notably the active tier's <c>activeNode</c>.
|
|
/// </summary>
|
|
public static string WithData(string status, params (string Name, string Status, Dictionary<string, object>? Data)[] entries)
|
|
{
|
|
var payload = new
|
|
{
|
|
status,
|
|
totalDurationMs = 1.5,
|
|
entries = entries.ToDictionary(
|
|
e => e.Name,
|
|
e => (object)new
|
|
{
|
|
status = e.Status,
|
|
description = $"{e.Name} says {e.Status}",
|
|
durationMs = 0.5,
|
|
data = e.Data,
|
|
},
|
|
StringComparer.Ordinal),
|
|
};
|
|
|
|
return JsonSerializer.Serialize(payload);
|
|
}
|
|
}
|
|
|
|
/// <summary>Builds <see cref="InstanceSnapshot"/> values for the pure-logic tests.</summary>
|
|
internal static class Snapshots
|
|
{
|
|
public static InstanceSnapshot Instance(
|
|
string name,
|
|
string? group = null,
|
|
string baseUrl = "http://host:8080",
|
|
bool hasActiveRole = true,
|
|
InstanceStatus? status = InstanceStatus.Up,
|
|
string? leader = null,
|
|
string clusterCheckName = "akka-cluster") =>
|
|
new()
|
|
{
|
|
Application = "App",
|
|
Instance = name,
|
|
Group = group,
|
|
BaseUrl = baseUrl,
|
|
HasActiveRole = hasActiveRole,
|
|
Status = status,
|
|
Report = leader is null ? null : ReportWithLeader(leader, clusterCheckName),
|
|
StaleAfter = TimeSpan.FromSeconds(45),
|
|
};
|
|
|
|
private static ZbHealthReport ReportWithLeader(string leader, string clusterCheckName)
|
|
{
|
|
var body = HealthBody.Ready("Healthy", (clusterCheckName, "Healthy", leader));
|
|
return JsonSerializer.Deserialize<ZbHealthReport>(body)!;
|
|
}
|
|
}
|