Joseph Doherty
329 lines
13 KiB
C#
329 lines
13 KiB
C#
using System.Diagnostics.Metrics;
|
|
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
|
|
|
namespace ZB.MOM.WW.MxGateway.Tests.Metrics;
|
|
|
|
public sealed class GatewayMetricsTests
|
|
{
|
|
/// <summary>Verifies that snapshot reflects all metric updates.</summary>
|
|
[Fact]
|
|
public void GetSnapshot_ReflectsSessionWorkerCommandEventAndFaultUpdates()
|
|
{
|
|
using GatewayMetrics metrics = new();
|
|
|
|
metrics.SessionOpened();
|
|
metrics.WorkerStarted(TimeSpan.FromMilliseconds(250));
|
|
metrics.CommandStarted("Register");
|
|
metrics.CommandSucceeded("Register", TimeSpan.FromMilliseconds(10));
|
|
metrics.CommandStarted("WriteSecured");
|
|
metrics.CommandFailed("WriteSecured", "AuthorizationFailed", TimeSpan.FromMilliseconds(12));
|
|
metrics.EventReceived("session-1", "OnDataChange");
|
|
metrics.EventReceived("session-1", "OnDataChange");
|
|
// GWC-30: the worker queue-depth gauge sums one live source per worker client, so the two
|
|
// registrations below stand in for two concurrent sessions holding 3 and 4 events.
|
|
using IDisposable workerDepthSourceA = metrics.RegisterWorkerEventQueueDepthSource(static () => 3);
|
|
using IDisposable workerDepthSourceB = metrics.RegisterWorkerEventQueueDepthSource(static () => 4);
|
|
// GWC-15: the gRPC stream queue-depth gauge sums live backlog sources at collection time
|
|
// rather than tracking a pushed running total. Register a source reporting 3.
|
|
using IDisposable backlogSource = metrics.RegisterEventStreamBacklogSource(static () => 3);
|
|
metrics.QueueOverflow("session-events");
|
|
metrics.Fault("CommandTimeout");
|
|
metrics.WorkerKilled("CommandTimeout");
|
|
metrics.WorkerStopped("Killed");
|
|
metrics.HeartbeatFailed("session-1");
|
|
metrics.StreamDisconnected("ClientCancelled");
|
|
metrics.SessionClosed();
|
|
|
|
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
|
|
Assert.Equal(0, snapshot.OpenSessions);
|
|
Assert.Equal(0, snapshot.WorkersRunning);
|
|
Assert.Equal(7, snapshot.WorkerEventQueueDepth);
|
|
Assert.Equal(3, snapshot.GrpcEventStreamQueueDepth);
|
|
Assert.Equal(1, snapshot.SessionsOpened);
|
|
Assert.Equal(1, snapshot.SessionsClosed);
|
|
Assert.Equal(2, snapshot.CommandsStarted);
|
|
Assert.Equal(1, snapshot.CommandsSucceeded);
|
|
Assert.Equal(1, snapshot.CommandsFailed);
|
|
Assert.Equal(2, snapshot.EventsReceived);
|
|
Assert.Equal(1, snapshot.QueueOverflows);
|
|
Assert.Equal(1, snapshot.Faults);
|
|
Assert.Equal(1, snapshot.WorkerKills);
|
|
Assert.Equal(1, snapshot.WorkerExits);
|
|
Assert.Equal(1, snapshot.HeartbeatFailures);
|
|
Assert.Equal(1, snapshot.StreamDisconnects);
|
|
Assert.Equal(1, snapshot.CommandFailuresByMethod["WriteSecured"]);
|
|
Assert.Equal(2, snapshot.EventsByFamily["OnDataChange"]);
|
|
Assert.Equal(2, snapshot.EventsBySession["session-1"]);
|
|
}
|
|
|
|
/// <summary>
|
|
/// GWC-30: the worker queue-depth gauge sums every registered source rather than holding a
|
|
/// single pushed scalar, so concurrent sessions add up instead of overwriting one another,
|
|
/// and a disposed registration (a worker client going away) drops out of the sum. Disposal
|
|
/// is idempotent because a client's dispose path can run twice.
|
|
/// </summary>
|
|
[Fact]
|
|
public void WorkerEventQueueDepthSources_SumAcrossRegistrationsAndDropOnDispose()
|
|
{
|
|
using GatewayMetrics metrics = new();
|
|
|
|
IDisposable firstSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 3);
|
|
using IDisposable secondSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 4);
|
|
|
|
Assert.Equal(7, metrics.GetSnapshot().WorkerEventQueueDepth);
|
|
|
|
firstSource.Dispose();
|
|
Assert.Equal(4, metrics.GetSnapshot().WorkerEventQueueDepth);
|
|
|
|
firstSource.Dispose();
|
|
Assert.Equal(4, metrics.GetSnapshot().WorkerEventQueueDepth);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A depth source reads a lock-free counter that a racing decrement can momentarily push
|
|
/// below zero, so the sum clamps each reading instead of rejecting it — the pull model has
|
|
/// no caller to throw back at.
|
|
/// </summary>
|
|
[Fact]
|
|
public void WorkerEventQueueDepthSources_ClampNegativeReadingsToZero()
|
|
{
|
|
using GatewayMetrics metrics = new();
|
|
|
|
using IDisposable negativeSource = metrics.RegisterWorkerEventQueueDepthSource(static () => -5);
|
|
using IDisposable positiveSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 4);
|
|
|
|
Assert.Equal(4, metrics.GetSnapshot().WorkerEventQueueDepth);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies the exported gauge keeps the name <c>mxgateway.events.worker_queue.depth</c> and
|
|
/// reports the summed sources, so the pull-model rework is invisible to exporters.
|
|
/// </summary>
|
|
[Fact]
|
|
public void WorkerEventQueueDepthGauge_ReportsSummedSources()
|
|
{
|
|
using GatewayMetrics metrics = new();
|
|
using MeterListener listener = new();
|
|
|
|
int? capturedDepth = null;
|
|
|
|
listener.InstrumentPublished = (instrument, meterListener) =>
|
|
{
|
|
if (ReferenceEquals(instrument.Meter, metrics.Meter)
|
|
&& instrument.Name == "mxgateway.events.worker_queue.depth")
|
|
{
|
|
meterListener.EnableMeasurementEvents(instrument);
|
|
}
|
|
};
|
|
listener.SetMeasurementEventCallback<int>(
|
|
(instrument, measurement, _, _) =>
|
|
{
|
|
if (ReferenceEquals(instrument.Meter, metrics.Meter)
|
|
&& instrument.Name == "mxgateway.events.worker_queue.depth")
|
|
{
|
|
capturedDepth = measurement;
|
|
}
|
|
});
|
|
listener.Start();
|
|
|
|
using IDisposable firstSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 5);
|
|
using IDisposable secondSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 6);
|
|
listener.RecordObservableInstruments();
|
|
|
|
Assert.Equal(11, capturedDepth);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The command counters are incremented with <see cref="Interlocked"/> rather than under the
|
|
/// process-wide metrics lock, so this asserts no increment is lost when every gRPC thread
|
|
/// records at once.
|
|
/// </summary>
|
|
[Fact]
|
|
public void CommandCounters_CountEveryConcurrentInvocation()
|
|
{
|
|
const int workers = 8;
|
|
const int perWorker = 500;
|
|
using GatewayMetrics metrics = new();
|
|
|
|
Parallel.For(0, workers, _ =>
|
|
{
|
|
for (int index = 0; index < perWorker; index++)
|
|
{
|
|
metrics.CommandStarted("Register");
|
|
metrics.CommandSucceeded("Register", TimeSpan.FromMilliseconds(1));
|
|
metrics.CommandFailed("WriteSecured", "AuthorizationFailed", TimeSpan.FromMilliseconds(1));
|
|
}
|
|
});
|
|
|
|
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
|
|
Assert.Equal(workers * perWorker, snapshot.CommandsStarted);
|
|
Assert.Equal(workers * perWorker, snapshot.CommandsSucceeded);
|
|
Assert.Equal(workers * perWorker, snapshot.CommandsFailed);
|
|
Assert.Equal(workers * perWorker, snapshot.CommandFailuresByMethod["WriteSecured"]);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that <see cref="GatewayMetrics.AlarmProviderSwitched"/> increments
|
|
/// <c>mxgateway.alarms.provider_switches</c> by one with the expected from/to/reason tags.
|
|
/// The listener filters by the specific <see cref="System.Diagnostics.Metrics.Meter"/> instance
|
|
/// to avoid cross-talk between parallel tests.
|
|
/// </summary>
|
|
[Fact]
|
|
public void AlarmProviderSwitched_IncrementsCounterWithExpectedTags()
|
|
{
|
|
using GatewayMetrics metrics = new();
|
|
using MeterListener listener = new();
|
|
|
|
long capturedValue = 0;
|
|
string? capturedFrom = null;
|
|
string? capturedTo = null;
|
|
string? capturedReason = null;
|
|
|
|
listener.InstrumentPublished = (instrument, meterListener) =>
|
|
{
|
|
if (ReferenceEquals(instrument.Meter, metrics.Meter)
|
|
&& instrument.Name == "mxgateway.alarms.provider_switches")
|
|
{
|
|
meterListener.EnableMeasurementEvents(instrument);
|
|
}
|
|
};
|
|
listener.SetMeasurementEventCallback<long>(
|
|
(instrument, measurement, tags, _) =>
|
|
{
|
|
if (!ReferenceEquals(instrument.Meter, metrics.Meter)
|
|
|| instrument.Name != "mxgateway.alarms.provider_switches")
|
|
{
|
|
return;
|
|
}
|
|
|
|
capturedValue += measurement;
|
|
foreach (KeyValuePair<string, object?> tag in tags)
|
|
{
|
|
switch (tag.Key)
|
|
{
|
|
case "from": capturedFrom = tag.Value as string; break;
|
|
case "to": capturedTo = tag.Value as string; break;
|
|
case "reason": capturedReason = tag.Value as string; break;
|
|
}
|
|
}
|
|
});
|
|
listener.Start();
|
|
|
|
metrics.AlarmProviderSwitched(1, 2, AlarmProviderSwitchReason.Failover);
|
|
|
|
Assert.Equal(1, capturedValue);
|
|
Assert.Equal("1", capturedFrom);
|
|
Assert.Equal("2", capturedTo);
|
|
Assert.Equal("failover", capturedReason);
|
|
Assert.Equal(1, metrics.GetSnapshot().AlarmProviderSwitchCount);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that <see cref="GatewayMetrics.AlarmProviderSwitched"/> updates the
|
|
/// <c>mxgateway.alarms.provider_mode</c> observable gauge to the <paramref name="toMode"/> value.
|
|
/// </summary>
|
|
[Fact]
|
|
public void AlarmProviderSwitched_UpdatesProviderModeGauge()
|
|
{
|
|
using GatewayMetrics metrics = new();
|
|
using MeterListener listener = new();
|
|
|
|
int? capturedMode = null;
|
|
|
|
listener.InstrumentPublished = (instrument, meterListener) =>
|
|
{
|
|
if (ReferenceEquals(instrument.Meter, metrics.Meter)
|
|
&& instrument.Name == "mxgateway.alarms.provider_mode")
|
|
{
|
|
meterListener.EnableMeasurementEvents(instrument);
|
|
}
|
|
};
|
|
listener.SetMeasurementEventCallback<int>(
|
|
(instrument, measurement, _, _) =>
|
|
{
|
|
if (ReferenceEquals(instrument.Meter, metrics.Meter)
|
|
&& instrument.Name == "mxgateway.alarms.provider_mode")
|
|
{
|
|
capturedMode = measurement;
|
|
}
|
|
});
|
|
listener.Start();
|
|
|
|
metrics.AlarmProviderSwitched(1, 2, AlarmProviderSwitchReason.Failover);
|
|
listener.RecordObservableInstruments();
|
|
|
|
Assert.Equal(2, capturedMode);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that <see cref="GatewayMetrics.HeartbeatFailed"/> increments
|
|
/// <c>mxgateway.heartbeats.failed</c> without emitting a <c>session_id</c> tag: the tag is
|
|
/// unbounded cardinality (every session mints a new exporter time series), so per-session
|
|
/// attribution is deliberately kept out of the exported counter.
|
|
/// </summary>
|
|
[Fact]
|
|
public void HeartbeatFailed_IncrementsCounterWithoutSessionIdTag()
|
|
{
|
|
using GatewayMetrics metrics = new();
|
|
using MeterListener listener = new();
|
|
|
|
long capturedValue = 0;
|
|
bool sawSessionIdTag = false;
|
|
|
|
listener.InstrumentPublished = (instrument, meterListener) =>
|
|
{
|
|
if (ReferenceEquals(instrument.Meter, metrics.Meter)
|
|
&& instrument.Name == "mxgateway.heartbeats.failed")
|
|
{
|
|
meterListener.EnableMeasurementEvents(instrument);
|
|
}
|
|
};
|
|
listener.SetMeasurementEventCallback<long>(
|
|
(instrument, measurement, tags, _) =>
|
|
{
|
|
if (!ReferenceEquals(instrument.Meter, metrics.Meter)
|
|
|| instrument.Name != "mxgateway.heartbeats.failed")
|
|
{
|
|
return;
|
|
}
|
|
|
|
capturedValue += measurement;
|
|
foreach (KeyValuePair<string, object?> tag in tags)
|
|
{
|
|
if (tag.Key == "session_id")
|
|
{
|
|
sawSessionIdTag = true;
|
|
}
|
|
}
|
|
});
|
|
listener.Start();
|
|
|
|
metrics.HeartbeatFailed("session-1");
|
|
|
|
Assert.Equal(1, capturedValue);
|
|
Assert.False(sawSessionIdTag);
|
|
Assert.Equal(1, metrics.GetSnapshot().HeartbeatFailures);
|
|
}
|
|
|
|
/// <summary>Verifies that removing session events only affects that session.</summary>
|
|
[Fact]
|
|
public void RemoveSessionEvents_RemovesOnlyThatSession()
|
|
{
|
|
using GatewayMetrics metrics = new();
|
|
|
|
metrics.EventReceived("session-1", "OnDataChange");
|
|
metrics.EventReceived("session-2", "OnWriteComplete");
|
|
metrics.RemoveSessionEvents("session-1");
|
|
|
|
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
|
|
Assert.Equal(2, snapshot.EventsReceived);
|
|
Assert.False(snapshot.EventsBySession.ContainsKey("session-1"));
|
|
Assert.Equal(1, snapshot.EventsBySession["session-2"]);
|
|
Assert.Equal(1, snapshot.EventsByFamily["OnDataChange"]);
|
|
Assert.Equal(1, snapshot.EventsByFamily["OnWriteComplete"]);
|
|
}
|
|
}
|