Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7da52b65b7 | |||
| 7b6dfba654 | |||
| 2b1efb5e50 | |||
| a390fe16e2 | |||
| c94c4d43c4 | |||
| df45cb4a37 |
@@ -6,7 +6,10 @@ $ErrorActionPreference = 'Stop'
|
||||
# those header stamps, so a regeneration on an off-pin machine would churn the tree and make
|
||||
# check-codegen Check 4 false-fail (or mask real drift under churn). Assert the exact versions
|
||||
# so a regen is deterministic. protoc itself is warn-only (source_code_info is normalized out of
|
||||
# the committed bindings), matching publish-client-proto-inputs.ps1.
|
||||
# the committed bindings), matching publish-client-proto-inputs.ps1. On Windows a plugin reports
|
||||
# its argv[0] name, so the banner carries a trailing `.exe` (e.g. "protoc-gen-go.exe v1.36.11");
|
||||
# Get-NormalizedToolVersion strips that suffix before the pin compare so Check 4 runs the same on
|
||||
# Windows as everywhere else.
|
||||
$PinnedProtocGenGoVersion = 'protoc-gen-go v1.36.11'
|
||||
$PinnedProtocGenGoGrpcVersion = 'protoc-gen-go-grpc 1.6.2'
|
||||
$PinnedProtocVersion = 'libprotoc 34.1'
|
||||
@@ -16,6 +19,14 @@ $protoRoot = Join-Path $repoRoot 'src\ZB.MOM.WW.MxGateway.Contracts\Protos'
|
||||
$outputRoot = Join-Path $PSScriptRoot 'internal\generated'
|
||||
$modulePath = 'gitea.dohertylan.com/dohertj2/mxaccessgw/clients/go/internal/generated'
|
||||
|
||||
function Get-NormalizedToolVersion {
|
||||
# On Windows a plugin reports its argv[0] name, so the banner carries an `.exe`
|
||||
# suffix ("protoc-gen-go.exe v1.36.11"). Strip it so the pin compare is
|
||||
# host-independent; the version part must still match exactly.
|
||||
param([string]$RawBanner)
|
||||
return ($RawBanner -replace '\.exe(?=\s)', '')
|
||||
}
|
||||
|
||||
function Resolve-Tool {
|
||||
# Resolve a codegen tool from PATH first (portable), then the documented Windows fallbacks,
|
||||
# instead of the previous hard-coded per-machine paths. See docs/ToolchainLinks.md.
|
||||
@@ -51,17 +62,17 @@ $protocGenGoGrpc = Resolve-Tool -Names @('protoc-gen-go-grpc', 'protoc-gen-go-gr
|
||||
|
||||
# Assert the pinned plugin versions before generating so Check 4 cannot false-fail (or mask drift)
|
||||
# on an off-pin machine. protoc is warn-only.
|
||||
$protocGenGoVersion = (& $protocGenGo --version 2>&1 | Out-String).Trim()
|
||||
$protocGenGoVersion = Get-NormalizedToolVersion (& $protocGenGo --version 2>&1 | Out-String).Trim()
|
||||
if ($protocGenGoVersion -ne $PinnedProtocGenGoVersion) {
|
||||
throw "protoc-gen-go reports '$protocGenGoVersion', but regeneration is pinned to '$PinnedProtocGenGoVersion'. " +
|
||||
"Install the pin: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11"
|
||||
}
|
||||
$protocGenGoGrpcVersion = (& $protocGenGoGrpc --version 2>&1 | Out-String).Trim()
|
||||
$protocGenGoGrpcVersion = Get-NormalizedToolVersion (& $protocGenGoGrpc --version 2>&1 | Out-String).Trim()
|
||||
if ($protocGenGoGrpcVersion -ne $PinnedProtocGenGoGrpcVersion) {
|
||||
throw "protoc-gen-go-grpc reports '$protocGenGoGrpcVersion', but regeneration is pinned to '$PinnedProtocGenGoGrpcVersion'. " +
|
||||
"Install the pin: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.6.2"
|
||||
}
|
||||
$protocVersion = (& $protoc --version 2>&1 | Out-String).Trim()
|
||||
$protocVersion = Get-NormalizedToolVersion (& $protoc --version 2>&1 | Out-String).Trim()
|
||||
if ($protocVersion -ne $PinnedProtocVersion) {
|
||||
Write-Warning "protoc reports '$protocVersion', pin is '$PinnedProtocVersion'. Descriptor comments are normalized out of the committed Go bindings, so patch drift is tolerated; keep CI on the pin."
|
||||
}
|
||||
|
||||
@@ -72,16 +72,18 @@ tasks.register('checkGeneratedClean') {
|
||||
group = 'verification'
|
||||
description = 'Fails if the committed generated Java tree differs from a fresh regeneration.'
|
||||
dependsOn 'generateProto'
|
||||
def generatedDir = 'clients/java/src/main/generated'
|
||||
def repoRoot = rootProject.projectDir.parentFile.parentFile
|
||||
// Project.exec was removed in Gradle 9; ProviderFactory.exec runs the git-status probe
|
||||
// lazily (captured at configuration time, evaluated in doLast) and works on both the
|
||||
// installed Gradle and Gradle 9.
|
||||
def gitStatus = providers.exec {
|
||||
workingDir(repoRoot)
|
||||
commandLine('git', 'status', '--porcelain', '--', generatedDir)
|
||||
ignoreExitValue = true
|
||||
}
|
||||
doLast {
|
||||
def generatedDir = 'clients/java/src/main/generated'
|
||||
def stdout = new ByteArrayOutputStream()
|
||||
def result = exec {
|
||||
workingDir = rootProject.projectDir.parentFile.parentFile
|
||||
commandLine 'git', 'status', '--porcelain', '--', generatedDir
|
||||
standardOutput = stdout
|
||||
ignoreExitValue = true
|
||||
}
|
||||
def dirty = stdout.toString().trim()
|
||||
def dirty = gitStatus.standardOutput.asText.get().trim()
|
||||
if (!dirty.isEmpty()) {
|
||||
throw new GradleException(
|
||||
"Generated Java is stale:\n${dirty}\n" +
|
||||
|
||||
@@ -260,9 +260,9 @@ Examples:
|
||||
|
||||
```bash
|
||||
mxgateway apikey init-db
|
||||
mxgateway apikey create-key --key-id ops.alice --display-name "Alice (ops)" --scopes invoke:read,invoke:write
|
||||
mxgateway apikey create-key --key-id area1.reader --display-name "Area 1 reader" --scopes invoke:read,metadata:read --read-subtree "Area1/*" --browse-subtree "Area1/*"
|
||||
mxgateway apikey create-key --key-id ops.temp --display-name "Temp contractor" --scopes invoke:read --expires 90d
|
||||
mxgateway apikey create-key --key-id ops.alice --display-name "Alice (ops)" --scopes session:open,invoke:read,invoke:write
|
||||
mxgateway apikey create-key --key-id area1.reader --display-name "Area 1 reader" --scopes session:open,invoke:read,metadata:read --read-subtree "Area1/*" --browse-subtree "Area1/*"
|
||||
mxgateway apikey create-key --key-id ops.temp --display-name "Temp contractor" --scopes session:open,invoke:read --expires 90d
|
||||
mxgateway apikey create-key --key-id team-a.svc --display-name "Team A service" --scopes session:open,invoke:read --dashboard-tags team-a
|
||||
mxgateway apikey create-key --key-id ops.audit --display-name "Audit window" --scopes metadata:read --expires 2027-01-01T00:00:00Z
|
||||
mxgateway apikey list-keys --json
|
||||
|
||||
@@ -245,8 +245,9 @@ cancelling, a detach-driven exit leaves the pill to the incoming subscription; w
|
||||
the pill therefore reports is the case it exists for — the channel completing under
|
||||
a page that is still watching.
|
||||
|
||||
`AlarmsPage` owns two loops of its own (the 3 s alarm poll and the provider-status
|
||||
badge) and bounds their drain at 5 seconds on dispose, for the same reason
|
||||
`AlarmsPage` owns two loops of its own (the 3 s alarm poll and the status feed that
|
||||
drives the provider badge and the truncated-snapshot banner) and bounds their drain
|
||||
at 5 seconds on dispose, for the same reason
|
||||
`DashboardPageBase` bounds its watch drain: both loops render through the renderer's
|
||||
dispatcher, and disposal can run on it. The two are drained concurrently, so the
|
||||
bound on disposal is 5 seconds in total rather than per loop — a wedged dispatcher
|
||||
@@ -285,11 +286,18 @@ Both seams consume the same producing services, so they share these cadences:
|
||||
- alarm publisher emits on each transition observed by the central monitor;
|
||||
- event publisher emits per event fanned by the session's `SessionEventDistributor`
|
||||
to its internal dashboard-mirror subscriber (independent of any gRPC `StreamEvents`);
|
||||
- the alarms page's provider-status badge resubscribes one second after its
|
||||
- the alarms page's status feed resubscribes one second after its
|
||||
`IGatewayAlarmService.StreamAsync` enumeration ends — the monitor completes a
|
||||
subscriber's stream when it falls behind and again when it restarts, both
|
||||
recoverable by resubscribing — and holds its last value in between. The page's
|
||||
alarm rows are independent of that stream and refresh on the 3 s poll.
|
||||
recoverable by resubscribing — and the badge and banner hold their last values in
|
||||
between. That feed carries both gateway-status frames: `provider_status` drives the
|
||||
badge, and `snapshot_status` drives the truncated-snapshot banner, so the caveat
|
||||
appears on the monitor's verdict change rather than up to three seconds later. Every
|
||||
subscriber is primed with a `snapshot_status` frame at open, so a page attaching
|
||||
mid-truncation needs no priming logic of its own. The page's alarm rows are
|
||||
independent of that stream and refresh on the 3 s poll, which also re-asserts the
|
||||
truncation verdict as its reconcile baseline — both sources read the same monitor
|
||||
verdict, and neither is synthesized page-side.
|
||||
|
||||
### Idle gating and snapshot cost
|
||||
|
||||
@@ -540,8 +548,11 @@ alarm-history store, so the page reflects only the live active set. The page is
|
||||
read-only; it does not acknowledge alarms. A provider-status badge tracks the
|
||||
central monitor's health from `IGatewayAlarmService.StreamAsync` in process — the
|
||||
alarm service is already a multi-subscriber fan-out, so the badge needs no SignalR
|
||||
client, no loopback socket, and no hub token — while the alarm rows themselves
|
||||
still come from the three-second poll. If `MxGateway:Alarms:Enabled` is
|
||||
client, no loopback socket, and no hub token — and the same subscription carries the
|
||||
`snapshot_status` frame behind the truncated-snapshot banner ("Alarm snapshot may be
|
||||
incomplete"), so a capped provider fetch is caveated the moment the monitor decides
|
||||
it. The alarm rows themselves still come from the three-second poll, which also
|
||||
re-asserts the truncation verdict as the reconcile baseline. If `MxGateway:Alarms:Enabled` is
|
||||
false the central monitor never starts, and the page says so instead of showing
|
||||
an empty list with no explanation.
|
||||
|
||||
|
||||
@@ -191,13 +191,13 @@
|
||||
private Task? _pollTask;
|
||||
|
||||
private DashboardAlarmProviderStatus _providerStatus = DashboardAlarmProviderStatus.Healthy;
|
||||
private Task? _providerStatusTask;
|
||||
private Task? _statusFeedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void OnInitialized()
|
||||
{
|
||||
_pollTask = PollLoopAsync();
|
||||
_providerStatusTask = ProviderStatusLoopAsync();
|
||||
_statusFeedTask = StatusFeedLoopAsync();
|
||||
}
|
||||
|
||||
private string? ProviderStatusTitle()
|
||||
@@ -210,8 +210,13 @@
|
||||
// The badge tracks the central monitor directly rather than looping back through
|
||||
// /hubs/alarms: the alarm service is an in-process multi-subscriber fan-out, so a
|
||||
// server-rendered page needs no SignalR client, no loopback socket and no auth token.
|
||||
// Alarm rows still come from the 3-second poll below — this loop only feeds the badge.
|
||||
private async Task ProviderStatusLoopAsync()
|
||||
// This loop feeds the two gateway-status indicators — the provider badge and the
|
||||
// truncation banner — from the feed's own status frames, so both move as soon as the
|
||||
// monitor's verdict changes instead of on the next 3-second tick. Alarm rows still come
|
||||
// from the poll below, which also re-asserts the truncation verdict as its reconcile
|
||||
// baseline: both sources read the same monitor verdict, so they cannot disagree for
|
||||
// longer than one tick, and neither one is synthesized here.
|
||||
private async Task StatusFeedLoopAsync()
|
||||
{
|
||||
while (!_cts.IsCancellationRequested)
|
||||
{
|
||||
@@ -221,16 +226,30 @@
|
||||
.StreamAsync(alarmFilterPrefix: null, _cts.Token)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
if (message.PayloadCase != AlarmFeedMessage.PayloadOneofCase.ProviderStatus)
|
||||
switch (message.PayloadCase)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
case AlarmFeedMessage.PayloadOneofCase.ProviderStatus:
|
||||
await InvokeAsync(() =>
|
||||
{
|
||||
_providerStatus = DashboardAlarmProviderStatus.FromFeed(message);
|
||||
StateHasChanged();
|
||||
}).ConfigureAwait(false);
|
||||
break;
|
||||
|
||||
await InvokeAsync(() =>
|
||||
{
|
||||
_providerStatus = DashboardAlarmProviderStatus.FromFeed(message);
|
||||
StateHasChanged();
|
||||
}).ConfigureAwait(false);
|
||||
// Every subscriber is primed with this frame at open, so a page that
|
||||
// attaches mid-truncation gets the caveat without waiting for an edge —
|
||||
// no page-side priming needed.
|
||||
case AlarmFeedMessage.PayloadOneofCase.SnapshotStatus:
|
||||
await InvokeAsync(() =>
|
||||
{
|
||||
_snapshotTruncated = message.SnapshotStatus.Truncated;
|
||||
StateHasChanged();
|
||||
}).ConfigureAwait(false);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
@@ -241,7 +260,8 @@
|
||||
{
|
||||
// The monitor completes a subscriber's stream when it falls behind, and
|
||||
// again when the monitor restarts. Both are recoverable by resubscribing;
|
||||
// the badge holds its last value in the meantime.
|
||||
// the badge and banner hold their last values in the meantime, and the
|
||||
// resubscribe is primed with the current ones.
|
||||
}
|
||||
|
||||
try
|
||||
@@ -321,7 +341,7 @@
|
||||
};
|
||||
}
|
||||
|
||||
// Fault handling sits inside the loop, matching ProviderStatusLoopAsync: a query or render
|
||||
// Fault handling sits inside the loop, matching StatusFeedLoopAsync: a query or render
|
||||
// fault on one tick is transient (a provider blip, a momentarily unavailable session), so it
|
||||
// is surfaced on the page and retried on the next tick rather than ending polling for the
|
||||
// life of the page. Cancellation is the only exit. The loop method itself therefore cannot
|
||||
@@ -398,6 +418,10 @@
|
||||
{
|
||||
DashboardAlarmQueryResult result = await LiveData.QueryAlarmsAsync(_cts.Token);
|
||||
_queryError = result.Error;
|
||||
// Kept alongside the feed's snapshot_status frame rather than replaced by it: this is
|
||||
// the reconcile baseline. Both read the same monitor verdict, so the poll can only
|
||||
// confirm what the frame already showed — but it also re-establishes the banner for a
|
||||
// page whose feed subscription is mid-resubscribe after the monitor dropped it.
|
||||
_snapshotTruncated = result.SnapshotTruncated;
|
||||
_workerPid = result.WorkerProcessId;
|
||||
_lastRefresh = DateTimeOffset.UtcNow;
|
||||
@@ -416,7 +440,7 @@
|
||||
// Drained together, not one after the other: the wedged dispatcher this bound exists
|
||||
// for blocks both loops at once, so sequential drains would time out twice and make
|
||||
// the real bound 10 seconds. DrainAsync tolerates a null task.
|
||||
await Task.WhenAll(DrainAsync(_pollTask), DrainAsync(_providerStatusTask))
|
||||
await Task.WhenAll(DrainAsync(_pollTask), DrainAsync(_statusFeedTask))
|
||||
.ConfigureAwait(false);
|
||||
|
||||
_cts.Dispose();
|
||||
|
||||
@@ -98,6 +98,8 @@ else
|
||||
<tr><th scope="row">Dashboard enabled</th><td>@Snapshot.Configuration.Dashboard.Enabled</td></tr>
|
||||
<tr><th scope="row">Anonymous localhost</th><td>@Snapshot.Configuration.Dashboard.AllowAnonymousLocalhost</td></tr>
|
||||
<tr><th scope="row">Snapshot interval</th><td>@Snapshot.Configuration.Dashboard.SnapshotIntervalMilliseconds ms</td></tr>
|
||||
<tr><th scope="row">Recent fault limit</th><td>@Snapshot.Configuration.Dashboard.RecentFaultLimit</td></tr>
|
||||
<tr><th scope="row">Recent session limit</th><td>@Snapshot.Configuration.Dashboard.RecentSessionLimit</td></tr>
|
||||
<tr><th scope="row">Show tag values</th><td>@Snapshot.Configuration.Dashboard.ShowTagValues</td></tr>
|
||||
<tr><th scope="row">Untagged session visibility</th><td>@Snapshot.Configuration.Dashboard.UntaggedSessionVisibility</td></tr>
|
||||
<tr><th scope="row">Worker protocol</th><td>@Snapshot.Configuration.Protocol.WorkerProtocolVersion</td></tr>
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Microsoft.AspNetCore.Components.Web.HtmlRendering;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Alarms;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
@@ -36,6 +40,8 @@ public sealed class AlarmsPageTruncationBannerTests
|
||||
{
|
||||
private const string BannerMarker = "Alarm snapshot may be incomplete";
|
||||
|
||||
private static readonly TimeSpan RenderWaitTimeout = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>A capped provider fetch puts the completeness caveat on the page.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -61,6 +67,120 @@ public sealed class AlarmsPageTruncationBannerTests
|
||||
Assert.Contains("Active Alarms", html, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The push path. A <c>snapshot_status</c> frame arriving on the page's
|
||||
/// in-process alarm-feed subscription raises the caveat on its own, with
|
||||
/// no poll tick behind it — the poll is a 3-second reconcile baseline, and
|
||||
/// an operator should not stare at an un-caveated alarm list for up to
|
||||
/// three seconds after the gateway has already decided the set is capped.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AlarmsPage_WhenFeedPushesTruncated_RaisesTheBannerWithoutAPollTick()
|
||||
{
|
||||
ScriptedAlarmFeed feed = new();
|
||||
await using ServiceProvider provider = BuildPushServices(feed);
|
||||
await using HtmlRenderer renderer = new(
|
||||
provider,
|
||||
provider.GetRequiredService<ILoggerFactory>());
|
||||
|
||||
HtmlRootComponent page = await renderer.Dispatcher.InvokeAsync(
|
||||
() => renderer.RenderComponentAsync<AlarmsPage>());
|
||||
|
||||
// The one poll answer this page will ever get said "complete", so everything
|
||||
// the banner does from here is the feed's doing.
|
||||
Assert.DoesNotContain(BannerMarker, await HtmlAsync(renderer, page), StringComparison.Ordinal);
|
||||
|
||||
await feed.PushAsync(SnapshotStatusFrame(truncated: true));
|
||||
|
||||
await WaitForHtmlAsync(
|
||||
renderer,
|
||||
page,
|
||||
html => html.Contains(BannerMarker, StringComparison.Ordinal),
|
||||
"banner to appear after a truncated snapshot_status frame");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The clearing edge, page-side. Absence authority comes back when the
|
||||
/// gateway says so; a banner that only ever went up would caveat the alarm
|
||||
/// list for the life of the circuit.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AlarmsPage_WhenFeedPushesComplete_ClearsTheBannerWithoutAPollTick()
|
||||
{
|
||||
ScriptedAlarmFeed feed = new();
|
||||
await using ServiceProvider provider = BuildPushServices(feed);
|
||||
await using HtmlRenderer renderer = new(
|
||||
provider,
|
||||
provider.GetRequiredService<ILoggerFactory>());
|
||||
|
||||
HtmlRootComponent page = await renderer.Dispatcher.InvokeAsync(
|
||||
() => renderer.RenderComponentAsync<AlarmsPage>());
|
||||
|
||||
await feed.PushAsync(SnapshotStatusFrame(truncated: true));
|
||||
await WaitForHtmlAsync(
|
||||
renderer,
|
||||
page,
|
||||
html => html.Contains(BannerMarker, StringComparison.Ordinal),
|
||||
"banner to appear before the clearing frame is pushed");
|
||||
|
||||
await feed.PushAsync(SnapshotStatusFrame(truncated: false));
|
||||
|
||||
await WaitForHtmlAsync(
|
||||
renderer,
|
||||
page,
|
||||
html => !html.Contains(BannerMarker, StringComparison.Ordinal),
|
||||
"banner to clear after a complete snapshot_status frame");
|
||||
}
|
||||
|
||||
private static AlarmFeedMessage SnapshotStatusFrame(bool truncated)
|
||||
{
|
||||
return new AlarmFeedMessage
|
||||
{
|
||||
SnapshotStatus = new AlarmSnapshotStatus { Truncated = truncated },
|
||||
};
|
||||
}
|
||||
|
||||
private static ServiceProvider BuildPushServices(ScriptedAlarmFeed feed)
|
||||
{
|
||||
ServiceCollection services = new();
|
||||
services.AddLogging();
|
||||
services.AddSingleton<IDashboardLiveDataService, SinglePollLiveDataService>();
|
||||
services.AddSingleton<IGatewayAlarmService>(feed);
|
||||
services.AddSingleton<IOptions<GatewayOptions>>(
|
||||
Options.Create(new GatewayOptions { Alarms = new AlarmsOptions { Enabled = true } }));
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private static Task<string> HtmlAsync(HtmlRenderer renderer, HtmlRootComponent page)
|
||||
{
|
||||
// Serialization has to happen on the renderer's dispatcher, and it reads the
|
||||
// component's current render tree — so it reflects renders the page's feed
|
||||
// loop queued after the initial quiescent render.
|
||||
return renderer.Dispatcher.InvokeAsync(page.ToHtmlString);
|
||||
}
|
||||
|
||||
private static async Task WaitForHtmlAsync(
|
||||
HtmlRenderer renderer,
|
||||
HtmlRootComponent page,
|
||||
Func<string, bool> predicate,
|
||||
string expectation)
|
||||
{
|
||||
Stopwatch elapsed = Stopwatch.StartNew();
|
||||
while (elapsed.Elapsed < RenderWaitTimeout)
|
||||
{
|
||||
if (predicate(await HtmlAsync(renderer, page)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(20));
|
||||
}
|
||||
|
||||
Assert.Fail($"Timed out after {RenderWaitTimeout.TotalSeconds:N0}s waiting for the {expectation}.");
|
||||
}
|
||||
|
||||
private static async Task<string> RenderAsync(bool snapshotTruncated)
|
||||
{
|
||||
ServiceCollection services = new();
|
||||
@@ -102,4 +222,93 @@ public sealed class AlarmsPageTruncationBannerTests
|
||||
WorkerProcessId: null,
|
||||
SnapshotTruncated: snapshotTruncated));
|
||||
}
|
||||
|
||||
// Answers exactly one poll — the inline first pass — and parks every later tick
|
||||
// until the page's disposal cancels it. The parking is what makes the push tests
|
||||
// measure the push: a second tick would re-assert the poll's own verdict, and
|
||||
// could either mask a banner the feed raised or raise one the feed did not.
|
||||
private sealed class SinglePollLiveDataService : IDashboardLiveDataService
|
||||
{
|
||||
private int _polls;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<DashboardLiveReadResult> ReadAsync(
|
||||
IReadOnlyCollection<string> tagAddresses,
|
||||
CancellationToken cancellationToken) =>
|
||||
Task.FromResult(DashboardLiveReadResult.Empty);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<DashboardAlarmQueryResult> QueryAlarmsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (Interlocked.Increment(ref _polls) > 1)
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return new DashboardAlarmQueryResult(
|
||||
Alarms: [],
|
||||
Error: null,
|
||||
WorkerProcessId: null,
|
||||
SnapshotTruncated: false);
|
||||
}
|
||||
}
|
||||
|
||||
// A hand-driven stand-in for the alarm feed: the test writes the frames the real
|
||||
// monitor would push. Unbounded and never completed, so a frame written before the
|
||||
// page's loop attaches is still delivered, and the loop never has to resubscribe.
|
||||
private sealed class ScriptedAlarmFeed : IGatewayAlarmService
|
||||
{
|
||||
private readonly Channel<AlarmFeedMessage> _frames =
|
||||
Channel.CreateUnbounded<AlarmFeedMessage>(new UnboundedChannelOptions
|
||||
{
|
||||
SingleReader = false,
|
||||
SingleWriter = false,
|
||||
});
|
||||
|
||||
/// <inheritdoc />
|
||||
public GatewayAlarmMonitorState State => GatewayAlarmMonitorState.Monitoring;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string? LastError => null;
|
||||
|
||||
/// <inheritdoc />
|
||||
public int? WorkerProcessId => null;
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyList<ActiveAlarmSnapshot> CurrentAlarms => [];
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool SnapshotTruncated { get; set; }
|
||||
|
||||
/// <summary>Pushes one frame onto the feed the page is subscribed to.</summary>
|
||||
/// <param name="message">The feed frame to deliver.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public ValueTask PushAsync(AlarmFeedMessage message) => _frames.Writer.WriteAsync(message);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<AlarmFeedMessage> StreamAsync(
|
||||
string? alarmFilterPrefix,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await foreach (AlarmFeedMessage message in _frames.Reader
|
||||
.ReadAllAsync(cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
yield return message;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<AcknowledgeAlarmReply> AcknowledgeAsync(
|
||||
AcknowledgeAlarmRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new AcknowledgeAlarmReply
|
||||
{
|
||||
CorrelationId = request.ClientCorrelationId,
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok },
|
||||
DiagnosticMessage = string.Empty,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,6 +73,26 @@ public sealed class SettingsPageTagVisibilityRenderTests
|
||||
Assert.Contains("Dashboard role mapping", html, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>RecentFaultLimit</c> and <c>RecentSessionLimit</c> reach the page. Non-default values
|
||||
/// prove the projection is plumbed through, not just that the defaults happen to render.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task SettingsPage_RendersRecentFaultAndSessionLimits()
|
||||
{
|
||||
string html = await RenderAsync(new DashboardOptions
|
||||
{
|
||||
RecentFaultLimit = 123,
|
||||
RecentSessionLimit = 456,
|
||||
});
|
||||
|
||||
Assert.Contains("Recent fault limit", html, StringComparison.Ordinal);
|
||||
Assert.Contains("123", html, StringComparison.Ordinal);
|
||||
Assert.Contains("Recent session limit", html, StringComparison.Ordinal);
|
||||
Assert.Contains("456", html, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private static async Task<string> RenderAsync(DashboardOptions dashboard)
|
||||
{
|
||||
EffectiveGatewayConfiguration configuration =
|
||||
|
||||
@@ -1453,8 +1453,15 @@ public sealed class WorkerPipeSessionTests
|
||||
/// <c>HeartbeatStuckCeiling</c> (75 s in production) keeps its activity
|
||||
/// timestamp fresh. This test compresses the clock — a 100 ms ceiling
|
||||
/// with a command in flight across a window many multiples longer — and
|
||||
/// models the pump refresh by continuously advancing the snapshot's
|
||||
/// <c>LastStaActivityUtc</c> while the command blocks. Contrast
|
||||
/// models the pump refresh with
|
||||
/// <see cref="FakeRuntimeSession.RefreshStaActivityOnCapture"/>, which
|
||||
/// stamps activity at every heartbeat capture exactly as the pump's
|
||||
/// per-iteration <c>MarkActivity()</c> does. The refresh has to be in
|
||||
/// effect from construction, not from the moment the command blocks:
|
||||
/// with a 50 ms grace, the idle window covering handshake and startup
|
||||
/// carries no correlation id for the watchdog to suppress on, so a fake
|
||||
/// whose activity timestamp is frozen at construction is reported
|
||||
/// <c>StaHung</c> before the scenario under test even starts. Contrast
|
||||
/// <see cref="RunAsync_WhenStaActivityIsStaleBeyondCeilingWithCommandInFlight_WritesWatchdogFault"/>,
|
||||
/// where a frozen timestamp beyond the ceiling correctly faults; here
|
||||
/// the refreshed timestamp must keep the fault suppressed and let the
|
||||
@@ -1469,6 +1476,13 @@ public sealed class WorkerPipeSessionTests
|
||||
FakeRuntimeSession runtime = new()
|
||||
{
|
||||
BlockDispatch = true,
|
||||
|
||||
// The pump refreshes STA activity on every wait iteration, so every
|
||||
// heartbeat capture on a healthy worker sees fresh activity — while
|
||||
// a command holds the STA and while it is idle alike. Armed before
|
||||
// RunAsync so the very first beat, sent as soon as the session is
|
||||
// Ready, is already covered.
|
||||
RefreshStaActivityOnCapture = true,
|
||||
};
|
||||
WorkerPipeSession session = CreatePipeSession(
|
||||
pipePair.WorkerStream,
|
||||
@@ -1490,49 +1504,48 @@ public sealed class WorkerPipeSessionTests
|
||||
runtime.DispatchStarted.Wait(TimeSpan.FromSeconds(5)),
|
||||
"The long command must reach the runtime and begin dispatch.");
|
||||
|
||||
// Model the pump refreshing STA activity on each wait iteration: keep
|
||||
// the snapshot's LastStaActivityUtc current while the command is in
|
||||
// flight.
|
||||
using CancellationTokenSource pumpRefresh = new();
|
||||
Task refreshLoop = Task.Run(
|
||||
async () =>
|
||||
{
|
||||
while (!pumpRefresh.IsCancellationRequested)
|
||||
{
|
||||
runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot(
|
||||
DateTimeOffset.UtcNow,
|
||||
pendingCommandCount: 1,
|
||||
outboundEventQueueDepth: 0,
|
||||
lastEventSequence: 0,
|
||||
currentCommandCorrelationId: "long-bulk-read"));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(20)).ConfigureAwait(false);
|
||||
}
|
||||
});
|
||||
// Publish the in-flight shape the heartbeat then reports for the whole
|
||||
// blocked window; only LastStaActivityUtc moves after this, refreshed by
|
||||
// the modelled pump at each capture.
|
||||
runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot(
|
||||
DateTimeOffset.UtcNow,
|
||||
pendingCommandCount: 1,
|
||||
outboundEventQueueDepth: 0,
|
||||
lastEventSequence: 0,
|
||||
currentCommandCorrelationId: "long-bulk-read"));
|
||||
|
||||
// Inspect a bounded number of frames over a window many multiples of the
|
||||
// 100 ms ceiling (at least 30 heartbeats at 20 ms ~ 600 ms). None may be
|
||||
// a WorkerFault while activity is continuously refreshed.
|
||||
const int framesToInspect = 30;
|
||||
for (int index = 0; index < framesToInspect; index++)
|
||||
int frameIndex = 0;
|
||||
for (; frameIndex < framesToInspect; frameIndex++)
|
||||
{
|
||||
WorkerEnvelope envelope = await pipePair.GatewayReader
|
||||
.ReadAsync(cancellation.Token);
|
||||
Assert.NotEqual(
|
||||
WorkerEnvelope.BodyOneofCase.WorkerFault,
|
||||
envelope.BodyCase);
|
||||
AssertNotFault(envelope, frameIndex);
|
||||
}
|
||||
|
||||
// Stop refreshing and release the command; its reply must be delivered
|
||||
// because the session never faulted (state stayed Ready).
|
||||
pumpRefresh.Cancel();
|
||||
await refreshLoop;
|
||||
// Release the command with the pump still running — as it is in
|
||||
// production while the reply is marshalled off the STA. The reply must
|
||||
// be delivered (the session never faulted, so its state stayed Ready),
|
||||
// and no frame on the way to it may be a fault either.
|
||||
runtime.ReleaseDispatch();
|
||||
|
||||
WorkerEnvelope reply = await ReadUntilAsync(
|
||||
pipePair.GatewayReader,
|
||||
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
|
||||
envelope => envelope.CorrelationId == "long-bulk-read",
|
||||
cancellation.Token);
|
||||
WorkerEnvelope reply;
|
||||
while (true)
|
||||
{
|
||||
WorkerEnvelope envelope = await pipePair.GatewayReader
|
||||
.ReadAsync(cancellation.Token);
|
||||
AssertNotFault(envelope, frameIndex++);
|
||||
if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerCommandReply
|
||||
&& envelope.CorrelationId == "long-bulk-read")
|
||||
{
|
||||
reply = envelope;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Assert.Equal(
|
||||
ProtocolStatusCode.Ok,
|
||||
reply.WorkerCommandReply.Reply.ProtocolStatus.Code);
|
||||
@@ -2217,6 +2230,26 @@ public sealed class WorkerPipeSessionTests
|
||||
/// <param name="predicate">Predicate to match against envelope.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>The matching envelope.</returns>
|
||||
/// <summary>
|
||||
/// Fails when the frame is a <c>WorkerFault</c>, naming the category and diagnostic message.
|
||||
/// A bare body-case comparison reports only "expected not WorkerFault", which says nothing
|
||||
/// about which watchdog or protocol path produced it — the one fact needed to tell a
|
||||
/// regression from a harness that mis-models the runtime.
|
||||
/// </summary>
|
||||
/// <param name="envelope">Frame read from the gateway end.</param>
|
||||
/// <param name="frameIndex">Ordinal of the frame within the inspected run.</param>
|
||||
private static void AssertNotFault(WorkerEnvelope envelope, int frameIndex)
|
||||
{
|
||||
if (envelope.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerFault)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Assert.Fail(
|
||||
$"Frame {frameIndex} is a WorkerFault ({envelope.WorkerFault.Category}): "
|
||||
+ envelope.WorkerFault.DiagnosticMessage);
|
||||
}
|
||||
|
||||
private static async Task<WorkerEnvelope> ReadUntilAsync(
|
||||
WorkerFrameReader reader,
|
||||
WorkerEnvelope.BodyOneofCase expectedBody,
|
||||
|
||||
@@ -127,11 +127,54 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
|
||||
});
|
||||
}
|
||||
|
||||
private bool refreshStaActivityOnCapture;
|
||||
|
||||
/// <summary>
|
||||
/// When set, <see cref="CaptureHeartbeat"/> stamps the snapshot's
|
||||
/// <c>LastStaActivityUtc</c> with the capture time and leaves every other field as the last
|
||||
/// <see cref="SetSnapshot"/> left it. Models a live STA whose pump calls
|
||||
/// <c>MarkActivity()</c> on each wait iteration (<c>StaRuntime.ThreadMain</c>), so a healthy
|
||||
/// worker is never captured stale — which a watchdog test needs to hold for the <em>whole</em>
|
||||
/// session, including the handshake window before any command exists for the watchdog to
|
||||
/// suppress on. A test-owned refresh loop cannot hold it: it is a thread-pool continuation
|
||||
/// racing a compressed grace, and the gap between this fake being constructed and that loop's
|
||||
/// first tick is already enough to look hung.
|
||||
/// </summary>
|
||||
public bool RefreshStaActivityOnCapture
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
return refreshStaActivityOnCapture;
|
||||
}
|
||||
}
|
||||
|
||||
set
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
refreshStaActivityOnCapture = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public WorkerRuntimeHeartbeatSnapshot CaptureHeartbeat()
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
if (refreshStaActivityOnCapture)
|
||||
{
|
||||
snapshot = new WorkerRuntimeHeartbeatSnapshot(
|
||||
DateTimeOffset.UtcNow,
|
||||
snapshot.PendingCommandCount,
|
||||
snapshot.OutboundEventQueueDepth,
|
||||
snapshot.LastEventSequence,
|
||||
snapshot.CurrentCommandCorrelationId,
|
||||
snapshot.StaCallInProgress);
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user